diff --git a/.github/workflows/sandboxd.yml b/.github/workflows/sandboxd.yml index 5e4c17792..78063b1e4 100644 --- a/.github/workflows/sandboxd.yml +++ b/.github/workflows/sandboxd.yml @@ -104,10 +104,12 @@ jobs: - name: Test TLS edge with Caddy working-directory: e2e + shell: bash env: CADDY_BIN: ${{ runner.temp }}/caddy run: | curl -fsSL --retry 3 https://github.com/caddyserver/caddy/releases/download/v2.11.4/caddy_2.11.4_linux_amd64.tar.gz -o "$RUNNER_TEMP/caddy.tar.gz" printf '%s %s\n' '8220d1f013b6f27510247b2360c9e0ca9f018feebd82515f07635318b34ff9777ccc8fd0b6e6f2486ce3a33fe389fbb7db12d05baa474f4587509fb4f5ebf1c9' "$RUNNER_TEMP/caddy.tar.gz" | sha512sum -c - tar -xzf "$RUNNER_TEMP/caddy.tar.gz" -C "$RUNNER_TEMP" caddy - GOWORK=off go test -race -count=1 -run TestCaddyTLSCluster -timeout 60s -v . + GOWORK=off go test -race -count=1 -run TestCaddyTLSCluster -timeout 60s -v . | tee "$RUNNER_TEMP/caddy-test.log" + ! grep -- '--- SKIP' "$RUNNER_TEMP/caddy-test.log" diff --git a/README.md b/README.md index 52c096aba..ab913bc52 100644 --- a/README.md +++ b/README.md @@ -7,20 +7,8 @@ for Go and Python. Warm claims are sub-millisecond; a pool miss clones from a golden snapshot in tens of milliseconds; cold boot is ~0.2–0.4 s on bare metal. **Documentation: [cocoonstack.github.io/sandbox](https://cocoonstack.github.io/sandbox/)** -(source in [`docs/`](docs/)). - -## Architecture - -``` -SDK (Go/Python) sandboxd (per node) guest microVM -client.New/new ─ HTTP ─► claim: warm pool / golden clone Cloud Hypervisor -exec/files/… ─ HTTP upgrade (kept) ─► byte relay ─ vsock ─► silkd :2048 - memberlist mesh: warm-count gossip, - MOVED-style redirect to the owning node -``` - -Cloud Hypervisor serves both network lanes. `net=none` has no NIC and uses -vsock-only I/O (hardened default); `net=egress` attaches a bridge/CNI NIC. +(source in [`docs/`](docs/)); the architecture overview is in +[`docs/index.md`](docs/index.md). ## Quick start @@ -140,7 +128,7 @@ The cmdline keys `sandbox-init` consumes are in the make help # every target; the ones below are the daily set make lint test # Rust: boot/init + silkd (fmt --check, clippy -D warnings, tests) make go-lint # Go: protocol/wire + sandboxd + sdk/go + e2e + mcp, GOOS linux AND darwin -make go-test # Go: go test -race across the Go modules +make go-test # Go: go test -race plus one pass of every benchmark, across the Go modules make sh-lint # shellcheck every tracked shell script make sandboxd # build dist/sandboxd make bench # claim-tier + data-plane benchmarks; removes every sbx-* VM diff --git a/boot/init/src/boot.rs b/boot/init/src/boot.rs index 3bf563181..2ea72e700 100644 --- a/boot/init/src/boot.rs +++ b/boot/init/src/boot.rs @@ -248,9 +248,9 @@ fn record_devices(ids: &[&str], found: &mut [Option]) { } fn record_serial(ids: &[&str], found: &mut [Option], serial: &str, device: &str) { - for (i, id) in ids.iter().enumerate() { - if found[i].is_none() && *id == serial && Path::new(device).exists() { - found[i] = Some(device.into()); + for (slot, id) in found.iter_mut().zip(ids) { + if slot.is_none() && *id == serial && Path::new(device).exists() { + *slot = Some(device.into()); } } } diff --git a/boot/init/src/cfg.rs b/boot/init/src/cfg.rs index 94784bf8d..4912e6497 100644 --- a/boot/init/src/cfg.rs +++ b/boot/init/src/cfg.rs @@ -61,8 +61,8 @@ pub fn parse(cmdline: &str) -> Result, String> { } } "cocoon.hostname" if !val.is_empty() => cfg.hostname = Some(val), - "sandbox.debug" => cfg.debug = debug_token(val), - "sandbox.trace" => cfg.trace = debug_token(val), + "sandbox.debug" => cfg.debug = flag_token(val), + "sandbox.trace" => cfg.trace = flag_token(val), "ip" => { if let Some(param) = parse_ip_param(val) { cfg.ips.push(param); @@ -85,7 +85,7 @@ pub fn parse(cmdline: &str) -> Result, String> { pub fn debug_requested(cmdline: &str) -> bool { params(cmdline) .rfind(|&(key, _)| key == "sandbox.debug") - .is_some_and(|(_, val)| debug_token(val)) + .is_some_and(|(_, val)| flag_token(val)) } /// Overlay mount data; index-based layer mountpoints keep a serial string out of lowerdir parsing. @@ -118,7 +118,7 @@ fn params(cmdline: &str) -> impl DoubleEndedIterator { .map(|tok| tok.split_once('=').unwrap_or((tok, ""))) } -fn debug_token(val: &str) -> bool { +fn flag_token(val: &str) -> bool { val.is_empty() || val == "1" } diff --git a/docs/cluster.md b/docs/cluster.md index 17db8d908..d4c332df8 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -31,8 +31,9 @@ Give every node a `mesh` block: join mesh: … connection refused`), so two nodes that name only each other cannot cold-start together: list the node's own `bind` alongside its peers and every node comes up in any order, joining whoever is already there -- `cluster_key` — optional base64 key (16/24/32 bytes) enabling gossip - encryption; all members must share it +- `cluster_key` — base64 key (16/24/32 bytes) enabling gossip encryption; + all members must share it. Required by `archive_failover`, + `checkpoint_peer_heal` and live moves Two constraints, fine for a homogeneous cluster: all nodes share the same `api_token` and tenant set (the SDK replays whichever token authorized a @@ -167,7 +168,7 @@ cap; to take no claims, cordon the node. | State | New claim, fork, branch | Existing claims | Root live move in | Adoption of archived claims | Refill and pool gossip | Survives restart and reboot | |---|---|---|---|---|---|---| -| serving | yes, under `max_claims`, `max_running`, the tenant's `max_claims`, `disk_reserve` and `memory_limit`; a 429 for a claim without volumes redirects to a peer | yes; a wake is bound by `max_running` and `disk_reserve`, and an archived claim's refused wake moves to a peer with room | yes, under the same caps and `move_concurrency` | yes; no claim cap binds, `max_running` binds at the wake | yes | — | +| serving | yes, unless an [admission refusal](sandboxd-api.md#admission-refusals) applies; a 429 for a claim without volumes redirects to a peer | yes; a wake is bound by `max_running` and `disk_reserve`, and an archived claim's refused wake moves to a peer with room | yes, under the same caps and `move_concurrency` | yes; no claim cap binds, `max_running` binds at the wake | yes | — | | cordoned | no: 429 `node cordoned`; peers stop redirecting here | yes | yes, under the same caps and `move_concurrency` | no; its own claims stay | no; warm VMs are destroyed | yes, as `/cordoned` | | draining | no: 429 `node draining`; peers stop redirecting here | yes | no: admission and commit refuse | no | no; warm VMs are destroyed | no | | draining with `evacuate=1` | no, as draining | yes until the claim's turn, then 503 until a peer holds it and 404 after; a claim that cannot move stays and serves | no | no | no | no: a restart stops the evacuation | @@ -695,8 +696,8 @@ that broadcast keeps its copy until the checkpoint TTL ages it out. A healed replica carries the source checkpoint's original `CreatedAt`, so it becomes eligible for expiry at the same instant everywhere; each node then removes it on its own hourly sweep, which is independently phased and retries on -failure. The window in which a deleted checkpoint stays branchable by an -id-holder is therefore normally `checkpoint_ttl_hours` plus the wait for the +failure. The window in which a deleted checkpoint stays branchable by its +tenant is therefore normally `checkpoint_ttl_hours` plus the wait for the next hourly sweep; a sweep that fails retries on a later one, so persistent sweep failure extends retention until one succeeds — the TTL is the eligibility point, not a hard ceiling. This is why heal *requires* a nonzero, fleet-matching diff --git a/docs/deploy.md b/docs/deploy.md index 842959f96..0eab49b15 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -129,8 +129,8 @@ the intersection of the two. | `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore loses what ran after the last committed replica: [the bound](cluster.md#running-claims-across-node-loss). The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 starts retiring its replicas while the registry remains enabled; see [Disabling archive failover](#disabling-archive-failover)) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `tun` (pool entry) | `false` | turns the [advanced network mode](advanced-network.md#configuration) on for the pool's claims; needs the node's `tun` block and the `none` lane. Config-owned like `egress`: `PUT /v1/pools` rejects it | -| `max_claims` | 0 (unlimited) | node-wide cap on live claims (to take none, [cordon](sandboxd-api.md#post-v1cordon) the node); claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim first tries a redirect to a warm peer, else to a peer that serves the pool; a volume claim answers the 429 with no redirect) | -| `max_running` | 0 (off) | node-wide cap on claims that hold a VM: not hibernated, not archived, not failed, with a wake in flight counted; claim, fork, branch and wake beyond it answer 429 and leave the claim as it was; a claim takes the same redirect as `max_claims`. Hibernated and archived claims hold no slot, so `max_claims` can follow disk while this follows RAM and CPU. Warm and reader VMs are not counted: keep `max_running` × the largest size plus every pool's warm target × its size within the node's RAM. A hibernated claim's wake cannot move to another node, so its 429 stays until a running claim sleeps or goes; with `archive_failover`, an archived claim's refused wake [moves to a peer with room](cluster.md#waking-an-archived-claim-on-a-full-node). `sandboxd_wakes_refused_total` counts the refusals. Lowering the cap stops nothing that runs. A [reload](#reloading-the-config) applies a change | +| `max_claims` | 0 (unlimited) | node-wide cap on live claims (to take none, [cordon](sandboxd-api.md#post-v1cordon) the node); claim/fork/branch requests beyond it answer 429 ([admission refusals](sandboxd-api.md#admission-refusals)) | +| `max_running` | 0 (off) | node-wide cap on claims that hold a VM: not hibernated, not archived, not failed, with a wake in flight counted; claim, fork, branch and wake beyond it answer 429 and leave the claim as it was ([admission refusals](sandboxd-api.md#admission-refusals)). Hibernated and archived claims hold no slot, so `max_claims` can follow disk while this follows RAM and CPU. Warm and reader VMs are not counted: keep `max_running` × the largest size plus every pool's warm target × its size within the node's RAM. A hibernated claim's wake cannot move to another node, so its 429 stays until a running claim sleeps or goes; with `archive_failover`, an archived claim's refused wake [moves to a peer with room](cluster.md#waking-an-archived-claim-on-a-full-node). `sandboxd_wakes_refused_total` counts the refusals. Lowering the cap stops nothing that runs. A [reload](#reloading-the-config) applies a change | | `move_concurrency` | 2 | node-wide cap on [live moves](cluster.md#moving-a-running-claim) in flight, counting moves this node sends and reservations it holds for moves to it; a move past it answers 429. Moves need `meta_store` with a `cell`, for the move record, and `mesh.cluster_key`, but not `archive_failover` or an S3 `checkpoint_store`: a node that sets this without them refuses to load, and one that leaves it unset without them cannot move (409). A move also needs a cocoon with live migration (v0.7.7 or newer; a master build is judged by the newest tag it reports), the Cloud Hypervisor fork at `35b098c65260` or newer (a VM started before the upgrade cannot move), a run dir that supports reflink such as XFS, nodes of the same CPU model, and the same path strings in cocoon's `root_dir` and `run_dir` on both nodes ([same paths](cluster.md#moving-a-running-claim)). Lowering the cap stops no move in flight. A [reload](#reloading-the-config) applies a change | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token; a config reload records as op `config_reload` with `changed`, a runtime volume change as op `volumes` with `added` or `removed`, and each ended flow of the [advanced network mode](advanced-network.md) as op `tun_flow` with `path` (`tun`), `tenant`, `proto` (`tcp`, `udp`, `icmp`), `dst_addr`, `decision` (`allow`, `deny`), `sent` and `received` bytes, `start`, `end` and the exit `profile`; a guest DNS-relay query records as op `dns` ([fields](egress.md#name-resolution-dns)), a cordon and its lift as ops `cordon` and `uncordon`, and a guest-port relay as op `port` with `port`. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `disk_reserve` | unset | `{"free": "100G", "dirs": ["/srv/sbx/run"]}`: while the filesystem of any listed directory (default `data_dir`) has less than `free` available, checked every 2 s, the node refuses new claims, forks, checkpoint claims, wakes of hibernated or archived sandboxes, hibernates, checkpoint captures and promotes with 429, and parks refill; the idle sweep and a lease-end archive leave their sandboxes running and retry on the next tick; `/v1/info` reports it as at capacity and the mesh sees no pools on it. Running sandboxes keep running. On an XFS project quota the free space is the project's. List cocoon's `run_dir` when it sits on another filesystem, since the copy-on-write disks grow there. Size `free` to at least two sandboxes' worst-case growth (copy-on-write disk plus memory file each) and keep the refusal above your disk alert's warning level, so the warning comes first: on a 640G project with 40G disks and 8G memory files, `100G` refuses at 84% used | @@ -138,7 +138,7 @@ the intersection of the two. | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview), no egress request in flight and no [advanced-network](advanced-network.md) packet carried for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh lease of the length the claim was granted (the server default of 5m when it asked for none); a lease moves only when a client calls [`/renew`](sandboxd-api.md#post-v1sandboxesidrenew). Keep `idle_hibernate_seconds` under the leases your claims use, or a woken sandbox that idles again is destroyed by the reap before it can hibernate, its archive already consumed by the wake; one still in use when its lease ends is destroyed the same way, unless it claimed `on_expire: archive`, which archives at the lease end whatever this setting. A call that wakes a hibernated claim past its lease, before the export, starts that fresh lease too. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived; the claim is then gone for good. On S3, chunks and manifests remain until [offline reclamation](#reclaiming-s3-storage). On archive this retention window replaces the live claim deadline; 0 clears the deadline so the archive is kept forever. An [evacuation](cluster.md#evacuating-a-node) keeps a longer live deadline, and a claim of a pool without an archive tier, whose `on_expire` is not `archive`, keeps its deadline until its first wake. Same node-wide/per-pool split | -| `archive_failover` | false | let another node adopt this node's archived claims once its node lease expires, and the running claims of pools with `replicate_seconds`: each archive and replication round also writes a row (token SHA-256, owner, epoch, checkpoint id, retention deadline, and the claim record with its env, `guest: false` values included, in plaintext) to the shared PostgreSQL, and a `Lookup` that reaches any node takes the claim over there (see [archived claims across node loss](cluster.md#archived-claims-across-node-loss)). Requires `meta_store` with a `cell` (rows are scoped by it, so meshes sharing one database never adopt from each other), a `checkpoint_store` of kind `s3` and a `mesh`, all checked at config load. The node also renews a lease row every 5 s (see [node lease](cluster.md#node-lease)). With the meta store down, archives, wakes, releases and env writes of archived claims, and env writes and renews of hibernated replicated claims, fail and retry, and replicated claims stop being served once the lease is overdue (see [running claims](cluster.md#running-claims-across-node-loss)); other running claims never touch it. Restart only | +| `archive_failover` | false | let another node adopt this node's archived claims once its node lease expires, and the running claims of pools with `replicate_seconds`: each archive and replication round also writes a row (token SHA-256, owner, epoch, checkpoint id, retention deadline, and the claim record with its env, `guest: false` values included, in plaintext) to the shared PostgreSQL, and a `Lookup` that reaches any node takes the claim over there (see [archived claims across node loss](cluster.md#archived-claims-across-node-loss)). Requires `meta_store` with a `cell` (rows are scoped by it, so meshes sharing one database never adopt from each other), a `checkpoint_store` of kind `s3` and a `mesh` with a `cluster_key` (a hand-off presents the claim's token to an address learned from gossip), all checked at config load. The node also renews a lease row every 5 s (see [node lease](cluster.md#node-lease)). With the meta store down, archives, wakes, releases and env writes of archived claims, and env writes and renews of hibernated replicated claims, fail and retry, and replicated claims stop being served once the lease is overdue (see [running claims](cluster.md#running-claims-across-node-loss)); other running claims never touch it. Restart only | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | | `pools[]` | — | warm pools, keyed by `(template, net, size)`. `warm` defaults to 4 only when both `warm` and `warm_max` are omitted; an explicit `warm: 0` is kept, so `warm: 0` under a `warm_max` starts the pool empty and grows it on demand, and `warm: 0` alone registers the key with no warm VMs (`PUT /v1/pools` never defaults `warm`); `net` is `none` (the default) or `egress`; `size` is a tier, below (default `small`). Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain, and their goldens are deleted once they are idle (`warm: 0` keeps the golden). This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. With a `meta_store` `cell`, the cell's set overrides it instead (see [shared pool set](#shared-pool-set)). Egress stays config-owned either way. A template re-pulled or re-imported under the same ref is picked up within a minute: the pool rebuilds its golden and replaces its warm VMs. See [state ownership](cluster.md#state-ownership) | @@ -183,9 +183,9 @@ connections to finish, and the next connection wakes the claim. A reader is a warm VM taken from the claim's own pool, so it boots the claim's image with the claim's layers, and it has no NIC (egress-lane claims never hibernate). Taking one removes a VM from the pool until the refill replaces it, -and an empty pool never cold-boots one: the call wakes the claim. A reader is never shared: it -serves one claim, and only the claim's own token reaches it. The guest's own -kernel does the parsing, as it does for the claim: +and an empty pool never cold-boots one: the call wakes the claim. Only the +claim's own token reaches its reader. The guest's own kernel does the parsing, +as it does for the claim: - a hibernated claim's copy-on-write disk is copied by cocoon (`vm disk attach --from-snapshot`) into the reader VM and hot-attached read-write; cocoon uses @@ -217,8 +217,8 @@ is destroyed. A wake waits for a reader only while cocoon copies the disk: milliseconds with a reflink, the time to copy the whole disk without one. A reader serves only the sleep it was built from: a later hibernate or archive gets a fresh one. The claim's own code runs in its reader with root, as it did -in the claim, and the reader serves one claim and is destroyed afterwards, so -the claim's bytes never reach a shared VM or the host kernel. +in the claim, and a reader is never shared, so the claim's bytes never reach a +shared VM or the host kernel. Whenever a reader cannot be built (no warm VM in the claim's pool, a failed attach, an archive without @@ -580,24 +580,11 @@ docs cover validation semantics and the per-VM knobs. ### Auth model -Three token kinds. The root `api_token` has full access — operators and -single-tenant deployments need nothing else. Tenant tokens, added through -the [tenant API](sandboxd-api.md#tenants-v1tenants) and kept in +A root `api_token` for operators, tenant tokens added through the +[tenant API](sandboxd-api.md#tenants-v1tenants) and kept in `/tenants.json` (or the [shared tenant database](#shared-tenant-database)), -create and manage their own resources: claims, forks, checkpoints, -promoted templates, and preview URLs are stamped with the tenant name; -sandbox and checkpoint listings filter to the caller's tenant, and a tenant -can delete only its own checkpoints and templates (root sees everything). -The [operator surfaces](sandboxd-api.md) stay root-only; a -tenant token there is authenticated but not authorized, so it answers 403 (a -wrong token stays 401). Per-sandbox -tokens are unchanged: whoever holds a sandbox's token drives that sandbox. -Fork children inherit the parent's tenant and count against its -`max_claims` (0 = unlimited, a per-node cap, so a tenant's cluster limit is -`max_claims` × nodes); a tenant at its cap gets 429 exactly like a node at -`max_claims`, and every usage event of a tenant's claim carries the tenant for -per-tenant billing. `config.json` has no `tenants` field: a node starts with -no tenants until the API adds them. +and one token per sandbox. What each may call is in the +[API reference](sandboxd-api.md#sandboxd-http-api). ## Running @@ -630,26 +617,9 @@ count): sandboxd then retires it and builds a new golden, and until that is ready, claims for that pool that find no warm VM cold-boot. If clones of the new golden fail the same way, it is retired again at most once every 30 s. -A minimal systemd unit (shipped as -[`packaging/sandboxd.service`](https://github.com/cocoonstack/sandbox/blob/main/packaging/sandboxd.service)): - -```ini -[Unit] -Description=sandboxd -Wants=network-online.target -After=network-online.target - -[Service] -ExecStart=/usr/local/bin/sandboxd -config /etc/sandboxd/config.json -Restart=on-failure -RestartSec=5s -KillMode=process -TimeoutStopSec=27min -Environment=SANDBOXD_LOG_LEVEL=info - -[Install] -WantedBy=multi-user.target -``` +The systemd unit is +[`packaging/sandboxd.service`](https://github.com/cocoonstack/sandbox/blob/main/packaging/sandboxd.service); +install it as shipped. Its `KillMode` and `TimeoutStopSec` are explained below. Each record carries its syslog level, so alerting keys on systemd priority: `journalctl -u sandboxd -p err` lists the errors and `-p warning` the @@ -853,7 +823,9 @@ only; computing durations or prices stays with the consumer. until it catches up. The journal append is unchanged, so a claim, release, hibernate or wake never waits on the database. - **Node:** rows carry the node's mesh `node_id` (its `bind` when unset), or - its host name without a mesh. The name must be unique within the cell. + its host name without a mesh. The name must be unique within the cell. Keep + the name stable: a renamed node lands its whole journal again under the new + name. - **Schema:** sandboxd creates the table at boot when it is missing; create it ahead with the same DDL when the node's role has no `CREATE` right: @@ -1130,7 +1102,7 @@ some of them on a node: |---|---|---| | `demo`, `smoke` | claim tiers and the verb surface | `sandboxd-e2e.sh` | | `volumesmoke` | shared read-only, writable, and live volumes | `sandboxd-e2e.sh` | -| `meshsmoke` | template routing on a two-node mesh without a shared checkpoint store, because [a shared store never routes](cluster.md#templates-on-a-cluster) | | +| `meshsmoke` | template routing on a two-node mesh without a shared checkpoint store, because [a shared store resolves templates locally](cluster.md#templates-on-a-cluster) | | | `crossnode` | a checkpoint claimed on another node through the shared store | | | `coldproof` | a cold claim, by the guest's own clock | | | `lifecycle` | idle → hibernate → archive → wake | `archive-e2e.sh` | diff --git a/docs/e2b.md b/docs/e2b.md index 90b103cf4..c7acfb536 100644 --- a/docs/e2b.md +++ b/docs/e2b.md @@ -2,7 +2,7 @@ The `e2b-rt` flavor carries [e2b](https://e2b.dev)'s `envd` alongside silkd, so an **unmodified e2b SDK** can drive a cocoon microVM; `e2b-ci` adds e2b's code -interpreter on top, for `@e2b/code-interpreter` and `e2b_code_interpreter`. It is the guest half of a +interpreter, for `@e2b/code-interpreter` and `e2b_code_interpreter`. It is the guest half of a compatibility layer whose other two halves live in [sandbox-operator](https://github.com/cocoonstack/sandbox-operator): the apiserver's e2b REST surface answers `Sandbox.create()`, and `envd-proxy` @@ -78,7 +78,9 @@ for the truth. commit's archive, checks its sha256, and applies the one local change listed in `os-image/e2b-ci/PATCHES.md`; e2b's license ships in the image under `/usr/share/doc/e2b-code-interpreter/`. Moving to a newer recipe is a bump of the -two `CI_RECIPE_*` args plus a refresh of the constraints files. On top of `e2b-rt`'s contents it runs a Jupyter server on +two `CI_RECIPE_*` args plus a refresh of the constraints files. It builds from +`base`, not on top of `e2b-rt`, with the same `envd` pin, guard unit and `user` +account as `e2b-rt`, and it runs a Jupyter server on loopback 8888 and the code-interpreter API on 49999, both enabled at boot, and the guard drops off-guest traffic to 49983, 49999 and 8888. Only the python kernel ships: the recipe installs its javascript, R and Java kernels from diff --git a/docs/egress.md b/docs/egress.md index 8ef17af3a..410eba6f2 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -178,8 +178,9 @@ presence; git keeps running as on any lane with a NIC. A golden built under another verdict is rebuilt, not adopted. Egress-lane sandboxes never archive and do not hibernate, fork, checkpoint, or -promote: cocoon resumes a guest before its fresh tap can be re-locked, so any -resume from a snapshot would open an unlocked-NIC window. Keeping the lane live +promote: cocoon resumes a restored or cloned guest inside the call, before +sandboxd can re-lock its tap, so any resume from a snapshot would open an +unlocked-NIC window. Keeping the lane live holds the lock unbroken from claim to release; the idle sweep skips the lane and those four verbs are refused (409) on it. @@ -383,9 +384,8 @@ gateway key per user, while the guest holds a placeholder. sets. Processes captured live in a snapshot (sessions, detached execs, units) keep the source's guest values in their own environment; a `guest: false` value never enters the guest, so no snapshot, checkpoint or - store object holds one. With [`archive_failover`](deploy.md#configuration), - the row of an archived or replicated claim carries the env, `guest: false` - values included, in plaintext in the shared PostgreSQL. + store object holds one; the `archive_failover` row is the exception + ([security](security.md#deployment-assumptions)). - **Audit.** An injection is recorded as the secret's name; the value is never logged, listed or returned. diff --git a/docs/openai-adapter.md b/docs/openai-adapter.md index 8c4e27f37..33fa22f15 100644 --- a/docs/openai-adapter.md +++ b/docs/openai-adapter.md @@ -44,5 +44,5 @@ ref, network lane (empty for the node's default, or `none`/`egress`), and `ttl_seconds`, which defaults to one hour: nothing renews a lease, and a session outlives the node's 5-minute default. The session state is JSON-serializable, so a run can be resumed against the same sandbox after a -process restart. Requires Python 3.10+ (the Agents SDK floor); the -underlying `cocoonsandbox` stays 3.9+. +process restart. Requires Python 3.10+, the floor of both the Agents SDK +and `cocoonsandbox`. diff --git a/docs/performance.md b/docs/performance.md index deb5009cd..a6234549f 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -6,7 +6,7 @@ Reproduce with `make bench` (tier definitions and the dated results log: [Benchmarks](benchmarks.md)), `scripts/sandboxd-e2e.sh` (claim/verb latencies print per run), and `scripts/boot-bench.sh` (boot phases). -**Bare metal** = a 16-core AMD (SVM) node, Ubuntu 24.04, kernel 6.x, local +**Bare metal** = a 16-core AMD (SVM) node, Ubuntu 24.04, kernel 6.17–7.0, local NVMe, KVM. **Nested** = a cloud VM with nested virtualization, never comparable with bare-metal claims ([why](benchmarks.md#the-three-claim-tiers)). diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 9ab6adfcf..47bcaae80 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -42,6 +42,24 @@ they reach its guest, so they share these answers: - 503 `tenant store down; retry` for an archived claim whose tenant the node has not cached while the meta store is down +## Admission refusals + +A claim, fork or checkpoint branch answers 429 when the node refuses it for +capacity, with the pool state unharmed. The causes: + +- the node is at [`max_claims`](deploy.md#configuration) or `max_running` +- the tenant the claim counts against is at its own `max_claims`: the caller's + for a claim or branch, the parent's for a fork +- the node is draining ([`POST /v1/drain`](#post-v1drain)) or cordoned + ([`POST /v1/cordon`](#post-v1cordon)) +- the disk is under `disk_reserve` +- memory is over `memory_limit` + +On a cluster, a claim without `volumes`, `no_redirect` or `require_promoted` +that meets one of these redirects to a peer with warm capacity or, failing +that, one that serves the pool; see [`POST /v1/claim`](#post-v1claim). Wake and +the guest-reaching verbs have their own 429 causes, listed with each. + ## POST /v1/claim Auth: `Authorization: Bearer ` (when configured). @@ -223,9 +241,8 @@ VM closes the devices. Redirects (mutually exclusive with the fields above) name peers to retry at — sent on a warm miss with warm peers, when the node lacks a golden for the key but gossip names a template owner, and when a claim without `volumes`, -`no_redirect` or `require_promoted` meets a 429 (the node at `max_claims` or -`max_running`, the calling tenant at its cap, the node draining or cordoned, the disk under -`disk_reserve`, or memory over `memory_limit`) and a peer reports warm capacity or, failing that, serves the pool: +`no_redirect` or `require_promoted` meets an [admission refusal](#admission-refusals) +and a peer reports warm capacity or, failing that, serves the pool: ```json {"redirect": ["10.0.0.6:7777", "10.0.0.7:7777"], @@ -258,9 +275,7 @@ busy — a live writer excludes every other claim for that name, live readers exclude a writer), or a `ro` claim against a writable image left dirty by an unclean `rw` release (needs recovery — one `rw` claim must replay and cleanly release before `ro` claims resume; see -[deploy](deploy.md#writable-dataset-volumes)); 429 node at `max_claims` or `max_running`, the -calling tenant at its own `max_claims`, the node draining or cordoned, the disk under -`disk_reserve`, or memory over `memory_limit`; 500 provisioning failed. +[deploy](deploy.md#writable-dataset-volumes)); 429 an [admission refusal](#admission-refusals); 500 provisioning failed. ## GET /v1/volumes @@ -424,19 +439,20 @@ a safe per-volume reason. Guest transport failures remain server errors; timeouts return 504 and cancellation returns 408. A mounted writable attachment marks the image dirty before the guest can write. -Live removal and release clear the marker only when the unmount observes and -normally unmounts the exact writable whole-device filesystem and finds no -residual device mounts. These checks read silkd's own mount namespace, so a -mount of the device in another namespace, or one moved there, is not seen. -Live removal also needs a confirmed host detach. Both -mount and superblock options must be writable; ext4's `emergency_ro` after a -filesystem error counts as read-only. A missing device or pre-unmounted -filesystem does not prove clean unmount. The marker remains on uncertainty and -blocks subsequent readers. Release waits up to 30 seconds for each writable -volume's unmount, and a guest that does not answer within that keeps every -remaining marker. Attach-only writers have no automatic dirty-marker -protection. The caller owns mounting, flushing, unmounting, recovery, and -fencing before reuse. +Live removal and release clear the marker only when the unmount of the mount at +the volume's path succeeds, the mount was the filesystem's root and writable in +both its mount and superblock options (ext4's `emergency_ro` after a filesystem +error counts as read-only), and no other mount of its device remains. Live +removal finds the device by its serial and refuses any other mount of it; +release identifies the mount by its path alone and does not check which device +it is. These checks read silkd's own mount namespace, so a mount of the device +in another namespace, or one moved there, is not seen. Live removal also needs +a confirmed host detach. A missing device or pre-unmounted filesystem does not +prove clean unmount. The marker remains on uncertainty and blocks subsequent +readers. Release waits up to 30 seconds for each writable volume's unmount, and +a guest that does not answer within that keeps every remaining marker. +Attach-only writers have no automatic dirty-marker protection. The caller owns +mounting, flushing, unmounting, recovery, and fencing before reuse. ### Partial failure and recovery @@ -599,8 +615,7 @@ an unknown `on_expire`; 401 bad api token; 403 the parent's tenant was removed (sandbox token); 404 unknown id or wrong sandbox token; 409 egress-lane or volume parent (neither forks, checkpoints, or promotes; see [egress](egress.md)), a failed parent, or an archived parent (`POST /wake` or an exec wakes it first); -429 node or the parent's tenant at `max_claims`, the node at `max_running`, the node draining or cordoned, the disk -under `disk_reserve`, or memory over `memory_limit`; 503 `lease lost; retry` for a replicated parent while its node's lease is overdue, `tenant store down; retry` for a sandbox token whose tenant the node has not cached while the meta store is down. +429 an [admission refusal](#admission-refusals); 503 `lease lost; retry` for a replicated parent while its node's lease is overdue, `tenant store down; retry` for a sandbox token whose tenant the node has not cached while the meta store is down. ## POST /v1/sandboxes/{id}/promote @@ -753,8 +768,10 @@ reach nothing on a pool with a policy. The API references a class and never defines one. A claim keeps the class its tenant had when it was made. A change applies to the next request: a removed or rotated token gets 401 at -once. Lowering `max_claims` below the live count refuses new claims and evicts -none. A removed tenant's claims keep running and stay listed for root +once. `max_claims` is a per-node cap (0 = unlimited), so a tenant's cluster +limit is `max_claims` × nodes; every usage event of a tenant's claim carries the +tenant, for per-tenant billing. Lowering `max_claims` below the live count +refuses new claims and evicts none. A removed tenant's claims keep running and stay listed for root (`GET /v1/sandboxes?tenant=`) until they expire or root releases them, but they cannot extend: renew and fork answer 403 to the sandbox's own token, a hibernated one is never archived, one archived before the removal is not @@ -937,7 +954,8 @@ for a replicated sandbox while its node's lease is overdue. Auth: node API token; body `{"ttl_seconds": 0, "no_redirect": false, "metadata": {}, "on_expire": "destroy", "env": {}}`. Claims a fresh sandbox branched from the checkpoint (a normal claim response, attributed to the caller); the checkpoint's recorded -key applies — the unguessable id is the capability to branch. The branch +key applies. A tenant token branches only its own tenant's checkpoints; any +other id answers 404, as for list and delete. The branch carries this request's `metadata`, `on_expire` and `env`, under the claim rules, never the source sandbox's. 400 metadata over a bound, an unknown `on_expire`, or a bad `env` entry. @@ -952,9 +970,7 @@ resolves every record) and answers with a redirect shaped like a warm-miss the claim. See the [placement lifecycle](cluster.md#checkpoints-on-a-cluster). 404 for an unknown checkpoint (locally, and after redirect and heal both -miss), 429 node -or calling tenant at `max_claims`, the node at `max_running`, the node draining or cordoned, the disk under -`disk_reserve`, or memory over `memory_limit`, 503 when the node's +miss), 429 an [admission refusal](#admission-refusals), 503 when the node's concurrent-heal cap is already full — retryable, and the response carries a `Retry-After` hint. @@ -1299,12 +1315,12 @@ claim of a replicating pool that is already open when the lease turns overdue is closed, whether or not it has a replica yet. A hibernated or archived sandbox wakes when the upgrade request arrives, before the `101`, -unless its last sleep is reader-safe, which the pool's [`reader`](deploy.md#reading-sleeping-claims-without-waking-them) setting provides. +unless its last sleep is reader-safe, which the pool's [`reader`](deploy.md#reading-sleeping-claims-without-waking-them) setting provides; +what a reader is, what it needs and when the call wakes instead are there. With it, the node answers the `101` before it touches the sandbox, reads the first frame, and serves `fs_stat`, `fs_list`, `fs_read` and `fs_pull` from a -reader VM, a warm VM of the sandbox's pool that mounts a copy of the sandbox's disk read-only: the sandbox stays -hibernated or archived, no `wake` usage event is recorded, and the read stamps -neither the idle nor the archive clock. An `info` frame wakes the sandbox, since a +reader VM: the sandbox stays hibernated or archived, no `wake` usage event is +recorded, and the read stamps neither the idle nor the archive clock. An `info` frame wakes the sandbox, since a reader would answer with its own processes and uptime. The first frame on the connection that is anything else, or any later one, wakes the sandbox and hands the same connection to its silkd with the unread bytes replayed, so a client sees one @@ -1313,13 +1329,9 @@ an archive), the connection's next read moves to a fresh reader of the new sleep when it wakes, the connection moves to the woken sandbox, so reads never come from a stale sleep. Pipelining a request behind a read that has not yet sent its terminal frame is unsupported: when that request moves the connection, the -reader closes and the rest of the read's reply is lost. Files -and symlinks resolve inside the sandbox's own filesystem as of the guest `sync` -that runs right before the last hibernate's pause; a write the guest made in -the instant between that `sync` and the pause is visible only after a wake. A -hibernate whose `sync` failed, or one made before the pool had `reader`, -is not served by a reader: its reads wake the sandbox. When no reader can be built (no warm VM in the pool, a failed attach, an archive from an older cocoon) the call wakes the sandbox as if the -feature were off. Because the `101` has gone out by then, a failure on this path +reader closes and the rest of the read's reply is lost. A read that no reader +can serve wakes the sandbox as if the feature were off. Because the `101` has +gone out by then, a failure on this path (unknown sandbox, a refused wake, an unreachable guest, an overdue node lease) arrives as a silkd `error` frame (`not_found` for an unknown sandbox, else `internal`) rather than an HTTP status, and the connection ends. A connection diff --git a/docs/sdk-python.md b/docs/sdk-python.md index b12a8a83d..254b042a6 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -17,7 +17,10 @@ their siblings), the claim env (`WithEnv`, `Env`/`SetEnv`/`PatchEnv`) and setting claim `metadata` or `on_expire`, and reading a claim's `net_route` remain Go-only, `Template.new` takes no `claim_ref`, and file and tar payloads are whole `bytes` values rather than streams (`read_file`/`pull` return them, `write_file`/`push` take them; the -wire is chunked, the caller's copy is not). Wire fidelity is +wire is chunked, the caller's copy is not). Two behaviors differ as well: +`Session.close()` and `Lsp.stop()` swallow `not_found` where Go returns it, and +a failed session command puts its merged output in `ExitError.stdout` with +`stderr` empty, where Go's `ExitError.Stderr` holds it. Wire fidelity is pinned by the shared protocol fixture corpus that the Rust guest, Go, and Python all round-trip in CI. @@ -209,7 +212,7 @@ Responses include `volumes_want`, `volumes`, `states`, and `converging`. `VolumePhase` and `VOLUME_ATTACHED`, `VOLUME_PENDING`, and `VOLUME_DETACHING` name the three states. A pending state with `converging=False` needs an explicit retry. `VolumeError`, exported from `cocoonsandbox` and `cocoonsandbox.errors`, -retains a complete failure view in `.response`. Complete replies update both +is a subclass of `APIError` and retains a complete failure view in `.response`. Complete replies update both public sets (`sb.volumes_want`, `sb.volumes`); incomplete errors preserve the last view. Serialize calls on one handle. See the [replacement tutorial](live-volumes.md) and [API recovery contract](sandboxd-api.md#partial-failure-and-recovery). diff --git a/docs/sdk.md b/docs/sdk.md index f89020ed1..7ada4ff17 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -103,8 +103,7 @@ client, err := sandbox.Connect("10.0.0.5:7777", sandbox.WithAPIToken(os.Getenv("SANDBOXD_TOKEN"))) ``` -- `Connect(addr, opts...)` — `addr` accepts a comma-separated seed list for - forward compatibility; the current version uses the first entry. +- `Connect(addr, opts...)` — `addr` is one node's address or origin. - `WithAPIToken(token)` — the node token: a root `api_token` (full access) or a tenant token (resource-creating verbs only; operator surfaces like `Info` answer it 403). On a cluster every node shares the same root token @@ -216,9 +215,10 @@ one already gone is not an error, and `Close` is bounded internally so it stays other call follows a move too: a 404 or a refused connection from the owner runs `Lookup` once, and a different owner replaces the old one. A call whose request is its whole input is sent again there: the control calls, `Run` -without `Stdin`, and the file, git, process and session calls that send no -data. `Run` with `Stdin`, `WriteFile`, `Push`, `OpenPty`, `Watch`, `DialPort`, `Lsp.Request`, -`Logs` and `Attach` fail once and the next call reaches the new owner. An +without `Stdin`, `WriteFile`, `OpenPty`, `Watch`, `DialPort`, `Lsp.Request`, +`Logs`, `Attach`, and the file, git, process and session calls. `Run` with +`Stdin` and `Push` stream the caller's reader, so they fail once and the next +call reaches the new owner. An error from the caller's own writer is returned as it is and never retried, and a released sandbox still answers 404. Volume sandboxes and egress-lane sandboxes cannot hibernate, fork, checkpoint, @@ -246,14 +246,8 @@ view, err = sb.PatchVolumes(ctx, sandbox.VolumePatch{Remove: []string{"imagenet" view, err = sb.SetVolumes(ctx, []sandbox.Volume{}) ``` -Python exposes the same operations and response fields: - -```python -view = sb.set_volumes([{"name": "imagenet", "mount": "/datasets/training"}]) -view = sb.get_volumes() -view = sb.patch_volumes(remove=["imagenet"]) -view = sb.set_volumes([]) # None is rejected. -``` +The [Python SDK](sdk-python.md#changing-live-volumes) exposes the same +operations and response fields. `Volume.AttachOnly` leaves mounting and unmounting to the caller: the entry in `Sandbox.Volumes` carries an empty `Mount`; see @@ -267,15 +261,12 @@ writer exclusion and dirty-marker limits. Each response contains `volumes_want`, `volumes`, `states`, and `converging`. Go exposes them as `SandboxVolumesResponse.VolumesWant`, `.Volumes`, `.States`, and `.Converging`. `VolumePhase` has `VolumeAttached`, `VolumePending`, and -`VolumeDetaching` constants. Python exposes `VolumePhase` and the corresponding -`VOLUME_ATTACHED`, `VOLUME_PENDING`, and `VOLUME_DETACHING` constants. Each state -can include an `error`. Methods use the client API token on the owning node +`VolumeDetaching` constants. Each state can include an `error`. Methods use the client API token on the owning node and update both public sets (`Sandbox.VolumesWant`, `Sandbox.Volumes`) from each complete response, including a partial failure. Serialize changes to one handle as with `Renew`. Returned views do not share the handle's slices or dictionaries. Go returns `*sandbox.VolumeError` with a `Response` alongside the returned view. -It wraps `*sandbox.APIError`, so `errors.As` can inspect the status. Python raises -`VolumeError`, a subclass of `APIError`, with `.response`. A response is absent +It wraps `*sandbox.APIError`, so `errors.As` can inspect the status. A response is absent when the reply carries no complete view (a malformed request, authentication, an unknown sandbox, or a non-JSON proxy reply). In that case the handle retains its last known sets. Retry an interrupted operation with the diff --git a/docs/security.md b/docs/security.md index f0527a24d..f77d2ee51 100644 --- a/docs/security.md +++ b/docs/security.md @@ -49,9 +49,8 @@ exposing any part of a deployment beyond a single trusted host. proxy's answers, and every credentialed call is journaled. Git auth tokens travel as in-memory headers, never guest disk. Secrets come from the host environment or a claim's `guest: false` env, never the config - file. That env stays in the node-local claim record; with - `archive_failover`, the row of an archived or replicated claim also carries - it, `guest: false` values included, in plaintext in the shared PostgreSQL. + file. That env stays in the node-local claim record, except in the + `archive_failover` row ([assumption 4](#deployment-assumptions)). - **Sandbox identity.** Every clone is reseeded (entropy, machine-id), so branches and forks never share an identity with their source. @@ -84,21 +83,14 @@ all of them. ## Token model -Three credentials, in descending scope ([API reference](sandboxd-api.md)): -the root `api_token` (operator surfaces, full access), tenant tokens -(resource-creating verbs, everything stamped and quota'd per tenant), and -per-sandbox tokens (that sandbox only — holding a handle amplifies to -nothing node-level). Two capability tokens ride on top: preview URLs are -HMAC-signed, expire with the claim's lease (an archived claim kept forever -has none, so its URL runs for the TTL it was minted with), and die with the -sandbox (no revocation list to leak); a checkpoint id is the unguessable capability to -branch it. On a cluster, deleting a checkpoint does not revoke that -capability fleet-wide the instant it runs: the delete is best-effort — -broadcast to every peer the node currently sees — so a peer that is offline -or partitioned at that moment keeps its own replica branchable until -`checkpoint_ttl_hours` ages it out ([placement lifecycle](cluster.md#checkpoints-on-a-cluster)). -Tenants are isolated at the API layer — listings filter, deletes answer 404 -rather than confirming existence, and operator surfaces answer tenants 403. +Three credentials in descending scope: the root `api_token`, tenant tokens +(isolated at the API layer) and per-sandbox tokens (that sandbox only, nothing +node-level); what each may call is in the +[API reference](sandboxd-api.md#sandboxd-http-api). A preview URL is a +capability on top, HMAC-signed with no revocation list. A checkpoint branches +only for its own tenant, and a delete does not remove it fleet-wide the +instant it runs +([why](cluster.md#delete-is-eventual-not-a-fleet-wide-revocation)). ## Dataset volumes diff --git a/docs/silkd.md b/docs/silkd.md index ecf69e16c..6478feffd 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -42,8 +42,10 @@ connection-bound verbs are `fs_watch`, `pty_open`, `lsp_request`, and `port_forward`. The authoritative wire contract is the shared fixture corpus in -`protocol/wire/fixtures/v1`, round-tripped by the Rust, Go, and Python test -suites — a frame only one side can parse fails CI. +`protocol/wire/fixtures/v1`, round-tripped by the Rust and Go test suites; the +Python suite round-trips the request frames and checks that each response +decodes to the fixture's `type` tag and `data` — a frame the Rust or Go side +cannot parse fails CI. ## Verbs @@ -55,8 +57,8 @@ suites — a frame only one side can parse fails CI. | fs | `fs_write {path, mode?}` (+`data`/`data_end` frames) / `fs_read` / `fs_list` / `fs_stat` / `fs_mkdir {parents?}` / `fs_rm {recursive?}` / `fs_rename {from, to}` | `fs_read` streams `data{data}`… → `done` for a regular file and answers `bad_request` for a directory, device or FIFO, `fs_list` streams 4096-entry `entries{entries}` batches → `done`, `fs_stat` answers one `stat{info}`, and the mutating verbs terminate with `done`. Streaming runs both directions; write commits atomically via temp+rename and inherits an overwritten file's mode and, under root, its owner | | tree | `fs_push {dest}` (+tar as `data` frames) / `fs_pull {path}` | whole trees as tar streams through the guest tar: `fs_pull` streams `data{data}`… → `done`, `fs_push` terminates with `done` | | search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files, and replace skips the same 8 MiB bound with a zero count. One match whose frame would pass the 8 MiB cap ends the find with an `error`. Replace is atomic per file, not per list: it stops with an `error` at the first missing, unreadable or non-UTF-8 file, and the files before it stay rewritten | -| watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects; watcher or delivery-queue overflow errors arrive as a terminal `error` instead of silently losing events | -| pty | `pty_open {cols, rows, cwd?, env?, user?}` / `pty_resize {pid, cols, rows}` | `pty_open` answers `started{pid}`, then a shell under a pseudo-terminal: output as `stdout` frames, input as `stdin` frames, `exit{code}` terminal; `pty_resize` answers `done`. PTYs register in the proc table like any exec | +| watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects or sends another request, which ends the watch with no terminal frame; watcher or delivery-queue overflow errors arrive as a terminal `error` instead of silently losing events | +| pty | `pty_open {cols, rows, cwd?, env?, user?}` / `pty_resize {pid, cols, rows}` | `pty_open` answers `started{pid}`, then a shell under a pseudo-terminal: output as `stdout` frames, input as `stdin` frames, `exit{code}` terminal; `pty_resize` answers `done`. PTYs register in the proc table like any exec. A client that disconnects hangs the shell up (SIGHUP, so it hangs up its jobs, as a closed ssh session does), and a shell still alive 2 s later is killed | | git | `git_clone {url, path, branch?, depth?, auth?}` / `git_status {path}` / `git_add {path, files}` / `git_commit {path, message, author}` / `git_push {path, auth?}` / `git_pull {path, auth?}` / `git_branch {path, action, name?}` | `git_status_result` (porcelain v2), `git_commit_result{hash}`, `git_branches{current, branches}` for `git_branch {action: "list"}`; every other verb terminates with `done`. A status past about 1 MiB of entries carries `truncated: true` with the head of the list, so one frame never exceeds the cap. `auth` is injected as an in-memory header, never written to guest disk | | port | `port_forward {port}` | relays guest TCP 127.0.0.1:port over this connection: `ready` once connected, then `data` both ways. Each direction ends on its own: `data_end` half-closes the guest socket, `done` reports the guest closing its write side, and the RPC ends once both have. A client that only drops the connection leaves the relay open until the guest side closes or writes. Works on both lanes — the no-network lane's only way in | | lsp | `lsp_start {language, root?}` / `lsp_request {server_id}` / `lsp_stop {server_id}` | a broker for the language server a flavor image ships: `lsp_start` spawns the argv named in `/etc/silkd/lsp.d/` (absent on the base image → `not_found` naming the flavor) and answers `lsp_started{server_id}`; `lsp_request` attaches the JSON-RPC byte stream (`ready`, then `data` both ways — silkd pipes bytes, it never parses LSP; `data_end` half-closes the server's stdin) and the stream ending reaps the server (v1 single-shot); `lsp_stop` kills it early, and a server nothing attaches within 5 minutes is reaped | diff --git a/e2e/cmd/androidsmoke/main.go b/e2e/cmd/androidsmoke/main.go index aef2bcc4a..e5d9aaeb4 100644 --- a/e2e/cmd/androidsmoke/main.go +++ b/e2e/cmd/androidsmoke/main.go @@ -102,18 +102,16 @@ func initTree(ctx context.Context, sb *sandbox.Sandbox) error { } func adbHandshake(ctx context.Context, sb *sandbox.Sandbox) error { - deadline := time.Now().Add(adbWait) - for { - reply, err := dialAdb(ctx, sb) - if err == nil { - fmt.Printf(" adb: port %d answered %s to CNXN\n", adbPort, reply) - return nil - } - if time.Now().After(deadline) { - return fmt.Errorf("adb port %d: %w", adbPort, err) - } - time.Sleep(2 * time.Second) + var reply string + err := harness.Poll(ctx, adbWait, 2*time.Second, func() (err error) { + reply, err = dialAdb(ctx, sb) + return err + }) + if err != nil { + return fmt.Errorf("adb port %d: %w", adbPort, err) } + fmt.Printf(" adb: port %d answered %s to CNXN\n", adbPort, reply) + return nil } // dialAdb sends one ADB CNXN; a CNXN or AUTH reply proves a live adbd. diff --git a/e2e/cmd/browsersmoke/main.go b/e2e/cmd/browsersmoke/main.go index fd8405da5..fe9afe4b3 100644 --- a/e2e/cmd/browsersmoke/main.go +++ b/e2e/cmd/browsersmoke/main.go @@ -73,21 +73,22 @@ func run(addr, token, template string) error { } func waitCDP(ctx context.Context, sb *sandbox.Sandbox) error { - deadline := time.Now().Add(cdpWait) - for { + err := harness.Poll(ctx, cdpWait, 2*time.Second, func() error { ver, err := cdpJSON(ctx, sb, "GET", "/json/version") - if err == nil { - if b, _ := ver["Browser"].(string); strings.Contains(b, "Chrome") { - fmt.Printf(" cdp: /json/version → %s (protocol %v)\n", b, ver["Protocol-Version"]) - return nil - } - err = fmt.Errorf("unexpected browser %v", ver["Browser"]) + if err != nil { + return err } - if time.Now().After(deadline) { - return fmt.Errorf("cdp never answered: %w", err) + b, _ := ver["Browser"].(string) + if !strings.Contains(b, "Chrome") { + return fmt.Errorf("unexpected browser %v", ver["Browser"]) } - time.Sleep(2 * time.Second) + fmt.Printf(" cdp: /json/version → %s (protocol %v)\n", b, ver["Protocol-Version"]) + return nil + }) + if err != nil { + return fmt.Errorf("cdp never answered: %w", err) } + return nil } // openTarget proves CDP accepts writes, not just liveness. diff --git a/e2e/cmd/coldproof/main.go b/e2e/cmd/coldproof/main.go index 92f78b180..a1d4f26cd 100644 --- a/e2e/cmd/coldproof/main.go +++ b/e2e/cmd/coldproof/main.go @@ -44,11 +44,13 @@ func run(addr, token, template string, n int) error { claim := time.Since(start) uptime, err := sb.Exec(ctx, "cat", "/proc/uptime") if err != nil { + _ = sb.Close() return fmt.Errorf("exec uptime: %w", err) } exec1 := time.Since(start) kernel, err := sb.Exec(ctx, "uname", "-r") if err != nil { + _ = sb.Close() return fmt.Errorf("exec uname: %w", err) } fmt.Printf("iter=%d claim=%.1fms first_exec_done=%.1fms guest_uptime=%ss guest_kernel=%s\n", diff --git a/e2e/cmd/demo/main.go b/e2e/cmd/demo/main.go index c6321a940..b172d864e 100644 --- a/e2e/cmd/demo/main.go +++ b/e2e/cmd/demo/main.go @@ -53,12 +53,15 @@ func run(addr, token, template, netShape, size string, n, ttl int, leak bool) er } claimed := time.Now() out, err := sb.Exec(ctx, "echo", "42") + if err == nil && out != "42\n" { + err = fmt.Errorf("got %q, want 42", out) + } if err != nil { + if !leak { + _ = sb.Close() + } return fmt.Errorf("exec in %s: %w", sb.ID, err) } - if out != "42\n" { - return fmt.Errorf("exec in %s: got %q, want 42", sb.ID, out) - } execDone := time.Now() fmt.Printf("iter=%d id=%s claim=%.1fms exec=%.1fms\n", i, sb.ID, claimed.Sub(start).Seconds()*1000, execDone.Sub(claimed).Seconds()*1000) diff --git a/e2e/cmd/guestserver/main.go b/e2e/cmd/guestserver/main.go index fa4958eef..66bb4f6f4 100644 --- a/e2e/cmd/guestserver/main.go +++ b/e2e/cmd/guestserver/main.go @@ -20,9 +20,8 @@ import ( // halfCloseSink greets, shuts its write side, and keeps reading what the client sends after. type halfCloseSink struct { - mu sync.Mutex - got string - done bool + mu sync.Mutex + got string } func (h *halfCloseSink) serve(addr string) { @@ -54,16 +53,13 @@ func (h *halfCloseSink) greetThenRead(conn net.Conn) { } rest, _ := io.ReadAll(tcp) h.mu.Lock() - h.got, h.done = string(rest), true + h.got = string(rest) h.mu.Unlock() } func (h *halfCloseSink) read() string { h.mu.Lock() defer h.mu.Unlock() - if !h.done { - return "" - } return h.got } diff --git a/e2e/cmd/lifecycle/main.go b/e2e/cmd/lifecycle/main.go index a629da757..69758f817 100644 --- a/e2e/cmd/lifecycle/main.go +++ b/e2e/cmd/lifecycle/main.go @@ -40,6 +40,7 @@ func run(addr, token, template, netShape string, wait time.Duration) error { if err != nil { return err } + defer func() { _ = sb.Close() }() id := sb.ID marker := fmt.Appendf(nil, "archive-marker-%d", time.Now().UnixNano()) if err = sb.WriteFile(ctx, markerPath, marker, nil); err != nil { @@ -47,7 +48,7 @@ func run(addr, token, template, netShape string, wait time.Duration) error { } fmt.Printf("claimed id=%s, marker written\n", id) - elapsed, err := waitArchived(ctx, client, 1, wait) + elapsed, err := waitArchived(ctx, client, wait) if err != nil { return err } @@ -81,8 +82,8 @@ func run(addr, token, template, netShape string, wait time.Duration) error { return nil } -// waitArchived returns how long the reaper's idle→hibernate→archive ladder took to archive n claims. -func waitArchived(ctx context.Context, client *sandbox.Client, n int, timeout time.Duration) (time.Duration, error) { +// waitArchived returns how long the reaper's idle→hibernate→archive ladder took to archive the claim. +func waitArchived(ctx context.Context, client *sandbox.Client, timeout time.Duration) (time.Duration, error) { start := time.Now() deadline := start.Add(timeout) for { @@ -90,12 +91,12 @@ func waitArchived(ctx context.Context, client *sandbox.Client, n int, timeout ti if err != nil { return 0, err } - if info.Archived == n { + if info.Archived == 1 { return time.Since(start), nil } if time.Now().After(deadline) { - return 0, fmt.Errorf("archived never reached %d within %s (hibernated=%d archived=%d claimed=%d)", - n, timeout, info.Hibernated, info.Archived, info.Claimed) + return 0, fmt.Errorf("archived never reached 1 within %s (hibernated=%d archived=%d claimed=%d)", + timeout, info.Hibernated, info.Archived, info.Claimed) } time.Sleep(500 * time.Millisecond) } diff --git a/e2e/cmd/meshsmoke/main.go b/e2e/cmd/meshsmoke/main.go index 0c8fb2ff3..ead451382 100644 --- a/e2e/cmd/meshsmoke/main.go +++ b/e2e/cmd/meshsmoke/main.go @@ -60,16 +60,12 @@ func run(addr, peer, token, template string) error { // Poll for gossip to reach the entry node; a premature claim cold-fails at A. var clone *sandbox.Sandbox - deadline := time.Now().Add(15 * time.Second) - for { + err = harness.Poll(ctx, 15*time.Second, 500*time.Millisecond, func() (err error) { clone, err = client.New(ctx, "mesh-tpl", sandbox.WithNetwork(sandbox.NetNone)) - if err == nil { - break - } - if time.Now().After(deadline) { - return fmt.Errorf("name-based claim of mesh-tpl via %s: %w", addr, err) - } - time.Sleep(500 * time.Millisecond) + return err + }) + if err != nil { + return fmt.Errorf("name-based claim of mesh-tpl via %s: %w", addr, err) } defer func() { _ = clone.Close() }() out, err := clone.Exec(ctx, "cat", "/root/marker") diff --git a/e2e/cmd/portsmoke/main.go b/e2e/cmd/portsmoke/main.go index b655f85ac..67f0b3ebd 100644 --- a/e2e/cmd/portsmoke/main.go +++ b/e2e/cmd/portsmoke/main.go @@ -120,15 +120,17 @@ func startGuestListener(ctx context.Context, sb *sandbox.Sandbox, listener strin if _, err := sb.Spawn(ctx, sandbox.Cmd{Argv: argv}); err != nil { return fmt.Errorf("spawn listener: %w", err) } - deadline := time.Now().Add(listenerReadyWait) - for time.Now().Before(deadline) { - out, execErr := sb.Exec(ctx, "sh", "-c", fmt.Sprintf("ss -ltn 'sport = :%d' | grep -c LISTEN", guestPort)) - if execErr == nil && strings.TrimSpace(out) != "0" { - return nil + bound := harness.Poll(ctx, listenerReadyWait, 500*time.Millisecond, func() error { + out, err := sb.Exec(ctx, "sh", "-c", fmt.Sprintf("ss -ltn 'sport = :%d' | grep -c LISTEN", guestPort)) + if err == nil && strings.TrimSpace(out) == "0" { + return errors.New("not listening") } - time.Sleep(500 * time.Millisecond) + return err + }) + if bound != nil { + return fmt.Errorf("listener never bound 127.0.0.1:%d", guestPort) } - return fmt.Errorf("listener never bound 127.0.0.1:%d", guestPort) + return nil } func stepHTTP(ctx context.Context, rt *harness.PortRelay, _ *sandbox.Sandbox) error { diff --git a/e2e/cmd/smoke/main.go b/e2e/cmd/smoke/main.go index 2bdf30025..4d8bbe3cd 100644 --- a/e2e/cmd/smoke/main.go +++ b/e2e/cmd/smoke/main.go @@ -510,16 +510,14 @@ func smokeTree(ctx context.Context, sb *sandbox.Sandbox) error { } // The staging unlink is asynchronous, so settle with a budget. var staging string - var stagingErr error - for range 20 { - staging, stagingErr = sb.Exec(ctx, "sh", "-c", "ls -a /root/tree") - if stagingErr == nil && !strings.Contains(staging, ".silkd-push-") { - break + if err := harness.Poll(ctx, 2*time.Second, 100*time.Millisecond, func() (err error) { + staging, err = sb.Exec(ctx, "sh", "-c", "ls -a /root/tree") + if err == nil && strings.Contains(staging, ".silkd-push-") { + err = errors.New("staging present") } - time.Sleep(100 * time.Millisecond) - } - if stagingErr != nil || strings.Contains(staging, ".silkd-push-") { - return fmt.Errorf("staging left behind: %q, %v", staging, stagingErr) + return err + }); err != nil { + return fmt.Errorf("staging left behind: %q, %v", staging, err) } var pulled bytes.Buffer @@ -535,12 +533,11 @@ func smokeTree(ctx context.Context, sb *sandbox.Sandbox) error { // smokePortForward proves sshd's banner arrives through DialPort and a dead port fails with not_found. func smokePortForward(ctx context.Context, sb *sandbox.Sandbox) error { // The step proves the relay, not sshd readiness, so the positive probe retries. - pc, err := sb.DialPort(ctx, 22) - for attempt := 0; err != nil && attempt < 100; attempt++ { - time.Sleep(100 * time.Millisecond) + var pc *sandbox.PortConn + if err := harness.Poll(ctx, 10*time.Second, 100*time.Millisecond, func() (err error) { pc, err = sb.DialPort(ctx, 22) - } - if err != nil { + return err + }); err != nil { return err } defer func() { _ = pc.Close() }() diff --git a/e2e/cmd/volumesmoke/main.go b/e2e/cmd/volumesmoke/main.go index 0a80d5bcb..f667a1fe1 100644 --- a/e2e/cmd/volumesmoke/main.go +++ b/e2e/cmd/volumesmoke/main.go @@ -69,12 +69,9 @@ func run(addr, token, template, volume, rwVolume, probe string, live bool) error _ = sbA.Close() return fmt.Errorf("claim at %s: %w", mountB, err) } - released := false defer func() { - if !released { - _ = sbA.Close() - _ = sbB.Close() - } + _ = sbA.Close() + _ = sbB.Close() }() if want := []sandbox.Volume{{Name: volume, Mount: mountA}}; !slices.Equal(sbA.Volumes, want) { return fmt.Errorf("first claim volumes %+v, want %+v", sbA.Volumes, want) @@ -114,7 +111,6 @@ func run(addr, token, template, volume, rwVolume, probe string, live bool) error if err := errors.Join(sbA.Close(), sbB.Close()); err != nil { return fmt.Errorf("release volume claims: %w", err) } - released = true fmt.Printf("VOLUME PASS concurrent_read_bytes=%d read_only=true released=true\n", len(outputs[0])) if live { if err := runLive(ctx, client, template, volume, probe, outputs[0]); err != nil { @@ -206,12 +202,7 @@ func runWritable(ctx context.Context, client *sandbox.Client, template, volume s return fmt.Errorf("writable claim: %w", err) } claimRW := time.Since(start) - live := writer - defer func() { - if live != nil { - _ = live.Close() - } - }() + defer func() { _ = writer.Close() }() if want := []sandbox.Volume{{Name: volume, Mount: mount, Mode: "rw"}}; !slices.Equal(writer.Volumes, want) { return fmt.Errorf("writable claim volumes %+v, want %+v", writer.Volumes, want) } @@ -231,14 +222,13 @@ func runWritable(ctx context.Context, client *sandbox.Client, template, volume s if err = writer.Close(); err != nil { return fmt.Errorf("release writable claim: %w", err) } - live = nil reader, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone), sandbox.WithVolumes(sandbox.Volume{Name: volume, Mount: mount})) if err != nil { return fmt.Errorf("read-only claim after the writer released: %w", err) } - live = reader + defer func() { _ = reader.Close() }() out, err := reader.Exec(ctx, "cat", path.Join(mount, file)) if err != nil { return fmt.Errorf("read back the written file: %w", err) @@ -249,7 +239,6 @@ func runWritable(ctx context.Context, client *sandbox.Client, template, volume s if err = reader.Close(); err != nil { return fmt.Errorf("release read-only claim: %w", err) } - live = nil fmt.Printf("VOLUME RW PASS volume=%s claim=%.1fms durable=true busy_refused=true\n", volume, float64(claimRW.Microseconds())/1000) return nil diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 0c987ca10..c130dd906 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -429,39 +429,67 @@ func TestVolumesEndToEnd(t *testing.T) { } } -func TestWritableVolumeEndToEnd(t *testing.T) { - scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") - st := startTenantStack(t, nil, - []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) +func TestWriterVolumeEndToEnd(t *testing.T) { + for _, tt := range []struct { + name string + volume sandbox.Volume + applied []string + released []string + marker bool + }{ + { + name: "writable", + volume: sandbox.Volume{Name: "scratch", Mount: "/datasets/rw", Mode: "rw"}, + applied: []string{"attach:scratch:rw", "mount:scratch:/datasets/rw:rw"}, + released: []string{"umount:/datasets/rw", "remove"}, + }, + { + name: "attach-only", + volume: sandbox.Volume{Name: "scratch", Mode: "rw", AttachOnly: true}, + applied: []string{"attach:scratch:rw"}, + released: []string{"remove"}, + marker: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") + st := startTenantStack(t, nil, + []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) - sb, err := st.client.New(t.Context(), "rt:24.04", - sandbox.WithVolumes(sandbox.Volume{Name: "scratch", Mount: "/datasets/rw", Mode: "rw"})) - if err != nil { - t.Fatalf("writable claim: %v", err) - } - if want := []sandbox.Volume{{Name: "scratch", Mount: "/datasets/rw", Mode: "rw"}}; !slices.Equal(sb.Volumes, want) { - t.Errorf("claim volumes %+v, want %+v", sb.Volumes, want) - } - applied := []string{"attach:scratch:rw", "mount:scratch:/datasets/rw:rw"} - if got := st.eng.volumeOpsLog(); !slices.Equal(got, applied) { - t.Errorf("engine ops %v, want %v", got, applied) - } - for _, requested := range []string{`{"name":"scratch"}`, `{"name":"scratch","mode":"rw"}`} { - if status, _ := rawClaim(t, st, requested); status != http.StatusConflict { - t.Errorf("claim %s under a live writer: %d, want 409", requested, status) - } - } + sb, err := st.client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(tt.volume)) + if err != nil { + t.Fatalf("%s claim: %v", tt.name, err) + } + if want := []sandbox.Volume{tt.volume}; !slices.Equal(sb.Volumes, want) { + t.Errorf("claim volumes %+v, want %+v", sb.Volumes, want) + } + if got := st.eng.volumeOpsLog(); !slices.Equal(got, tt.applied) { + t.Errorf("engine ops %v, want %v", got, tt.applied) + } + if tt.marker { + assertNoDirtyMarker(t, scratch, "apply") + } + for _, requested := range []string{`{"name":"scratch"}`, `{"name":"scratch","mode":"rw"}`} { + if status, _ := rawClaim(t, st, requested); status != http.StatusConflict { + t.Errorf("claim %s under a live %s writer: %d, want 409", requested, tt.name, status) + } + } - if err := sb.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - want := slices.Concat(applied, []string{"umount:/datasets/rw", "remove"}) - if got := st.eng.volumeOpsLog(); !slices.Equal(got, want) { - t.Errorf("engine ops after release %v, want %v", got, want) + if err := sb.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + want := slices.Concat(tt.applied, tt.released) + if got := st.eng.volumeOpsLog(); !slices.Equal(got, want) { + t.Errorf("engine ops after release %v, want %v", got, want) + } + if tt.marker { + assertNoDirtyMarker(t, scratch, "release") + } + }) } } -func TestVolumeModeWireShape(t *testing.T) { +func TestVolumeWireShape(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") st := startTenantStack(t, nil, []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) @@ -480,6 +508,16 @@ func TestVolumeModeWireShape(t *testing.T) { `{"name":"scratch","mount":"/datasets/x"}`, map[string]any{"name": "scratch", "mount": "/datasets/x"}, }, + { + "attach-only omits the mount", + `{"name":"scratch","mode":"rw","attach_only":true}`, + map[string]any{"name": "scratch", "mode": "rw", "attach_only": true}, + }, + { + "eager claim is unchanged", + `{"name":"scratch","mode":"rw"}`, + map[string]any{"name": "scratch", "mount": "/volumes/scratch", "mode": "rw"}, + }, } { t.Run(tt.name, func(t *testing.T) { status, claimed := rawClaim(t, st, tt.requested) @@ -602,10 +640,7 @@ func TestClaimEnvEndToEnd(t *testing.T) { if gotA, gotB := seen(a), seen(b); gotA != "Bearer tenant-a" || gotB != "Bearer tenant-b" { t.Fatalf("injected %q / %q, want each claim's own value", gotA, gotB) } - st.eng.mu.Lock() - guestEnv := st.eng.guestEnvs[sockA] - st.eng.mu.Unlock() - if guestEnv != "AGENT_MODE=\"on\"\n" { + if guestEnv := st.eng.guestEnv(sockA); guestEnv != "AGENT_MODE=\"on\"\n" { t.Errorf("guest env file %q, want only the guest entry", guestEnv) } if status, body := rawJSON(t, st, http.MethodPut, "/v1/sandboxes/"+idA+"/env", `{"env":{"GW_KEY":{"value":"Bearer rotated","guest":false}}}`); status != http.StatusNoContent { @@ -636,9 +671,7 @@ func TestClaimEnvThroughTheSDK(t *testing.T) { if err != nil { t.Fatalf("socket: %v", err) } - st.eng.mu.Lock() - defer st.eng.mu.Unlock() - return st.eng.guestEnvs[sock] + return st.eng.guestEnv(sock) } readEnv := func(sb *sandbox.Sandbox, want map[string]sandbox.EnvVar) { t.Helper() @@ -753,88 +786,6 @@ func TestForkClaimRefPrefixWireShape(t *testing.T) { } } -func TestAttachOnlyVolumeEndToEnd(t *testing.T) { - scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") - st := startTenantStack(t, nil, - []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) - - sb, err := st.client.New(t.Context(), "rt:24.04", - sandbox.WithVolumes(sandbox.Volume{Name: "scratch", Mode: "rw", AttachOnly: true})) - if err != nil { - t.Fatalf("attach-only claim: %v", err) - } - if want := []sandbox.Volume{{Name: "scratch", Mode: "rw", AttachOnly: true}}; !slices.Equal(sb.Volumes, want) { - t.Errorf("claim volumes %+v, want %+v", sb.Volumes, want) - } - applied := []string{"attach:scratch:rw"} - if got := st.eng.volumeOpsLog(); !slices.Equal(got, applied) { - t.Errorf("engine ops %v, want %v", got, applied) - } - assertNoDirtyMarker(t, scratch, "apply") - for _, requested := range []string{`{"name":"scratch"}`, `{"name":"scratch","mode":"rw"}`} { - if status, _ := rawClaim(t, st, requested); status != http.StatusConflict { - t.Errorf("claim %s under a live attach-only writer: %d, want 409", requested, status) - } - } - - if err := sb.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - want := slices.Concat(applied, []string{"remove"}) - if got := st.eng.volumeOpsLog(); !slices.Equal(got, want) { - t.Errorf("engine ops after release %v, want %v", got, want) - } - assertNoDirtyMarker(t, scratch, "release") -} - -func TestAttachOnlyVolumeWireShape(t *testing.T) { - scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") - st := startTenantStack(t, nil, - []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) - for _, tt := range []struct { - name string - request string - want map[string]any - }{ - { - "attach-only omits the mount", - `{"template":"rt:24.04","volumes":[{"name":"scratch","mode":"rw","attach_only":true}]}`, - map[string]any{"name": "scratch", "mode": "rw", "attach_only": true}, - }, - { - "eager claim is unchanged", - `{"template":"rt:24.04","volumes":[{"name":"scratch","mode":"rw"}]}`, - map[string]any{"name": "scratch", "mount": "/volumes/scratch", "mode": "rw"}, - }, - } { - t.Run(tt.name, func(t *testing.T) { - status, body := rawJSON(t, st, http.MethodPost, "/v1/claim", tt.request) - if status != http.StatusOK { - t.Fatalf("claim: %d %s, want 200", status, body) - } - var reply map[string]any - if err := json.Unmarshal(body, &reply); err != nil { - t.Fatalf("decode claim %s: %v", body, err) - } - entries, _ := reply["volumes"].([]any) - if len(entries) != 1 { - t.Fatalf("reply volumes %v, want one entry", reply["volumes"]) - } - entry, _ := entries[0].(map[string]any) - if !maps.Equal(entry, tt.want) { - t.Errorf("reply volume %v, want %v", entry, tt.want) - } - var claimed rawClaimResponse - if err := json.Unmarshal(body, &claimed); err != nil { - t.Fatalf("decode claim %s: %v", body, err) - } - if err := st.client.Attach(st.addr, claimed.ID, claimed.Token).Close(); err != nil { - t.Fatalf("release: %v", err) - } - }) - } -} - func TestDirtyVolumeRefusesReader(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") if err := os.WriteFile(scratch+".dirty", nil, 0o600); err != nil { diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 2e7d57908..f8ac3c6f7 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -254,6 +254,24 @@ func (f *fakeEngine) volumeOpsLog() []string { return slices.Clone(f.volumeOps) } +func (f *fakeEngine) guestEnv(sock string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.guestEnvs[sock] +} + +func (f *fakeEngine) setReaderRoot(root string) { + f.mu.Lock() + defer f.mu.Unlock() + f.readerRoot = root +} + +func (f *fakeEngine) readerCounts() (started, attached int) { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.readerStarts), len(f.readerAttaches) +} + func (f *fakeEngine) create(name string) (string, error) { f.mu.Lock() defer f.mu.Unlock() diff --git a/e2e/internal/harness/harness.go b/e2e/internal/harness/harness.go index ebf9e6ec8..e6743012e 100644 --- a/e2e/internal/harness/harness.go +++ b/e2e/internal/harness/harness.go @@ -10,6 +10,7 @@ import ( "net/http" "strconv" "strings" + "time" sandbox "github.com/cocoonstack/sandbox/sdk/go" ) @@ -93,6 +94,24 @@ func Claim(ctx context.Context, addr, token, template string, opts ...sandbox.Op return client, sb, nil } +func Poll(ctx context.Context, budget, every time.Duration, try func() error) error { + deadline := time.Now().Add(budget) + for { + err := try() + if err == nil { + return nil + } + if time.Now().After(deadline) { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(every): + } + } +} + // HTTPOverPort sends one HTTP/1.1 request over the port relay and returns a 200 body; the Host stays "localhost" because Chrome's DevTools allowlist accepts it but not proxied vhosts. func HTTPOverPort(ctx context.Context, sb *sandbox.Sandbox, port uint16, method, path string, body []byte) ([]byte, error) { pc, err := sb.DialPort(ctx, port) diff --git a/e2e/internal/harness/harness_test.go b/e2e/internal/harness/harness_test.go index 755334d1d..29d3292d0 100644 --- a/e2e/internal/harness/harness_test.go +++ b/e2e/internal/harness/harness_test.go @@ -2,6 +2,8 @@ package harness import ( "bufio" + "context" + "errors" "fmt" "io" "net" @@ -51,6 +53,31 @@ func TestPortRelayPreservesGreetingAndHalfClose(t *testing.T) { } } +func TestPoll(t *testing.T) { + errTry := errors.New("not yet") + + calls := 0 + if err := Poll(t.Context(), time.Second, time.Millisecond, func() error { + calls++ + if calls < 3 { + return errTry + } + return nil + }); err != nil || calls != 3 { + t.Errorf("success: err %v calls %d, want nil and 3", err, calls) + } + + if err := Poll(t.Context(), 5*time.Millisecond, time.Millisecond, func() error { return errTry }); !errors.Is(err, errTry) { + t.Errorf("exhausted: err %v, want the last try error", err) + } + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := Poll(ctx, time.Minute, time.Minute, func() error { return errTry }); !errors.Is(err, context.Canceled) { + t.Errorf("canceled: err %v, want context.Canceled", err) + } +} + func serveGuestFirstHalfClose(listener net.Listener) error { conn, err := listener.Accept() if err != nil { diff --git a/e2e/reader_test.go b/e2e/reader_test.go index 05d29b0a1..df447fc98 100644 --- a/e2e/reader_test.go +++ b/e2e/reader_test.go @@ -21,9 +21,7 @@ func TestReaderServesAHibernatedClaimUntilAWriteWakesIt(t *testing.T) { if err := os.WriteFile(filepath.Join(root, "work", "out.txt"), []byte("result"), 0o600); err != nil { t.Fatalf("setup: %v", err) } - st.eng.mu.Lock() - st.eng.readerRoot = root - st.eng.mu.Unlock() + st.eng.setReaderRoot(root) sb, err := st.client.New(t.Context(), testKey.Template) if err != nil { t.Fatalf("New: %v", err) @@ -44,9 +42,7 @@ func TestReaderServesAHibernatedClaimUntilAWriteWakesIt(t *testing.T) { if !hibernated(t, st, sb.ID) { t.Fatal("reading a file woke the claim") } - st.eng.mu.Lock() - started, attached := len(st.eng.readerStarts), len(st.eng.readerAttaches) - st.eng.mu.Unlock() + started, attached := st.eng.readerCounts() if started != 1 || attached != 1 { t.Errorf("%d readers started over %d snapshot copies, want one of each", started, attached) } diff --git a/mcp/node_info_test.go b/mcp/node_info_test.go index 562a41af6..609aa94da 100644 --- a/mcp/node_info_test.go +++ b/mcp/node_info_test.go @@ -3,13 +3,14 @@ package main import ( "encoding/json/v2" "net/http" - "net/http/httptest" - "strings" "testing" ) func TestNodeInfoReportsCapacityState(t *testing.T) { - node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/info" { + return false + } _ = json.MarshalWrite(w, map[string]any{ "pools": []any{}, "claimed": 2, @@ -18,13 +19,8 @@ func TestNodeInfoReportsCapacityState(t *testing.T) { "at_capacity": true, "at_capacity_reason": "not enough memory", }) - })) - t.Cleanup(node.Close) - - srv, err := newServer(strings.TrimPrefix(node.URL, "http://"), "", "rt:24.04") - if err != nil { - t.Fatalf("newServer: %v", err) - } + return true + }) text, err := toolNodeInfo(t.Context(), srv, nil) if err != nil { t.Fatalf("toolNodeInfo: %v", err) diff --git a/mcp/server_test.go b/mcp/server_test.go index a57122dba..9fbfee915 100644 --- a/mcp/server_test.go +++ b/mcp/server_test.go @@ -14,7 +14,6 @@ import ( "sync" "sync/atomic" "testing" - "time" "github.com/cocoonstack/sandbox/protocol/wire" "github.com/cocoonstack/sandbox/sdk/go/silkd/silkdtest" @@ -53,14 +52,14 @@ func TestServeSpeaksMCP(t *testing.T) { t.Fatalf("got %d replies, want 8 (notification unanswered)", len(replies)) } - initResult := replies[1]["result"].(map[string]any) + initResult := as[map[string]any](t, replies[1]["result"]) if initResult["protocolVersion"] != protocolVersion { t.Errorf("initialize protocolVersion = %v", initResult["protocolVersion"]) } - if info := initResult["serverInfo"].(map[string]any); info["version"] != version { + if info := as[map[string]any](t, initResult["serverInfo"]); info["version"] != version { t.Errorf("initialize serverInfo = %v, want version %q", info, version) } - tools := replies[2]["result"].(map[string]any)["tools"].([]any) + tools := as[[]any](t, as[map[string]any](t, replies[2]["result"])["tools"]) if len(tools) != len(toolSpecs()) { t.Errorf("tools/list: %d tools, want %d", len(tools), len(toolSpecs())) } @@ -76,7 +75,7 @@ func TestServeSpeaksMCP(t *testing.T) { t.Errorf("release: %q", got) } - execReply := replies[6]["result"].(map[string]any) + execReply := as[map[string]any](t, replies[6]["result"]) if execReply["isError"] != true { t.Errorf("exec after release: %+v, want isError", execReply) } @@ -114,7 +113,7 @@ func TestExecKeepsOutputWhenTheGuestDrops(t *testing.T) { `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"create_sandbox","arguments":{}}}`, `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"exec","arguments":{"sandbox_id":"sb_1","command":"yes"}}}`, ) - execResult := replies[2]["result"].(map[string]any) + execResult := as[map[string]any](t, replies[2]["result"]) text := toolText(t, replies[2]) if execResult["isError"] != true || !strings.Contains(text, `"stdout":"partial"`) || !strings.Contains(text, `"error"`) { t.Errorf("exec reply %v: want isError with the partial stdout and an error field", text) @@ -179,9 +178,13 @@ func TestToolRepliesReplaceInvalidUTF8(t *testing.T) { func TestCloseBoxesWaitsForEveryReleaseOnce(t *testing.T) { var releases atomic.Int32 + var arrived sync.WaitGroup + arrived.Add(3) + gate := make(chan struct{}) node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/release") { - time.Sleep(200 * time.Millisecond) + arrived.Done() + <-gate releases.Add(1) } w.WriteHeader(http.StatusNoContent) @@ -196,14 +199,20 @@ func TestCloseBoxesWaitsForEveryReleaseOnce(t *testing.T) { srv.trackBox(srv.client.Attach(addr, id, "tok")) } var wg sync.WaitGroup + called := make(chan struct{}, 2) for range 2 { wg.Go(func() { + called <- struct{}{} srv.closeBoxes() if got := releases.Load(); got != 3 { t.Errorf("releases = %d after closeBoxes returned, want 3", got) } }) } + <-called + <-called + arrived.Wait() + close(gate) wg.Wait() } @@ -240,14 +249,23 @@ func TestCappedOutputMarksTruncation(t *testing.T) { } } +func as[T any](t *testing.T, v any) T { + t.Helper() + got, ok := v.(T) + if !ok { + t.Fatalf("%v is %T, want %T", v, v, got) + } + return got +} + func toolText(t *testing.T, resp map[string]any) string { t.Helper() res, ok := resp["result"].(map[string]any) if !ok { t.Fatalf("reply without result: %+v", resp) } - content := res["content"].([]any) - return fmt.Sprintf("%v", content[0].(map[string]any)["text"]) + content := as[[]any](t, res["content"]) + return fmt.Sprintf("%v", as[map[string]any](t, content[0])["text"]) } func newTestServer(t *testing.T, extra func(http.ResponseWriter, *http.Request) bool) *server { @@ -313,7 +331,7 @@ func serveLines(t *testing.T, srv *server, lines ...string) map[int]map[string]a if err := json.Unmarshal(line, &resp); err != nil { t.Fatalf("decode reply %q: %v", line, err) } - replies[int(resp["id"].(float64))] = resp + replies[int(as[float64](t, resp["id"]))] = resp } return replies } diff --git a/mcp/tools.go b/mcp/tools.go index 1b7a74cfa..c76e09cd1 100644 --- a/mcp/tools.go +++ b/mcp/tools.go @@ -96,6 +96,49 @@ func (a sandboxArg) id() string { return a.SandboxID } type sandboxIDer interface{ id() string } +type cmdArgs struct { + sandboxArg + Command string `json:"command"` + Cwd string `json:"cwd"` +} + +type killArgs struct { + sandboxArg + PID uint32 `json:"pid"` + Signal int32 `json:"signal"` +} + +type logsArgs struct { + sandboxArg + PID uint32 `json:"pid"` +} + +type writeFileArgs struct { + sandboxArg + Path string `json:"path"` + Content string `json:"content"` +} + +type pathArgs struct { + sandboxArg + Path string `json:"path"` +} + +type forkArgs struct { + sandboxArg + Count int `json:"count"` +} + +type checkpointArgs struct { + sandboxArg + Name string `json:"name"` +} + +type promoteArgs struct { + sandboxArg + TemplateName string `json:"template_name"` +} + // cappedOutput keeps the first execOutputCap bytes: an agent that tails a firehose must not grow this process. type cappedOutput struct { strings.Builder @@ -148,20 +191,6 @@ func toolCreateSandbox(ctx context.Context, s *server, raw jsontext.Value) (stri return jsonText(map[string]any{"sandbox_id": sb.ID, "deadline": sb.Deadline}), nil } -type cmdArgs struct { - sandboxArg - Command string `json:"command"` - Cwd string `json:"cwd"` -} - -func parseCommand(s *server, raw jsontext.Value) (cmdArgs, *sandbox.Sandbox, error) { - args, sb, err := s.parseAndBox[cmdArgs](raw) - if err == nil && args.Command == "" { - return args, nil, errors.New("command must not be empty") - } - return args, sb, err -} - func toolExec(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseCommand(s, raw) if err != nil { @@ -208,12 +237,6 @@ func toolPs(ctx context.Context, s *server, raw jsontext.Value) (string, error) return jsonText(procs), nil } -type killArgs struct { - sandboxArg - PID uint32 `json:"pid"` - Signal int32 `json:"signal"` -} - func toolKill(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[killArgs](raw) if err != nil { @@ -225,11 +248,6 @@ func toolKill(ctx context.Context, s *server, raw jsontext.Value) (string, error return "killed", nil } -type logsArgs struct { - sandboxArg - PID uint32 `json:"pid"` -} - func toolLogs(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[logsArgs](raw) if err != nil { @@ -247,12 +265,6 @@ func toolLogs(ctx context.Context, s *server, raw jsontext.Value) (string, error return jsonText(out), nil } -type writeFileArgs struct { - sandboxArg - Path string `json:"path"` - Content string `json:"content"` -} - func toolWriteFile(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[writeFileArgs](raw) if err != nil { @@ -264,11 +276,6 @@ func toolWriteFile(ctx context.Context, s *server, raw jsontext.Value) (string, return fmt.Sprintf("wrote %d bytes to %s", len(args.Content), args.Path), nil } -type pathArgs struct { - sandboxArg - Path string `json:"path"` -} - func toolReadFile(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[pathArgs](raw) if err != nil { @@ -295,11 +302,6 @@ func toolListDir(ctx context.Context, s *server, raw jsontext.Value) (string, er return jsonText(entries), nil } -type forkArgs struct { - sandboxArg - Count int `json:"count"` -} - func toolFork(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[forkArgs](raw) if err != nil { @@ -317,11 +319,6 @@ func toolFork(ctx context.Context, s *server, raw jsontext.Value) (string, error return jsonText(map[string]any{"sandbox_ids": ids}), nil } -type checkpointArgs struct { - sandboxArg - Name string `json:"name"` -} - func toolCheckpoint(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[checkpointArgs](raw) if err != nil { @@ -383,11 +380,6 @@ func toolHibernate(ctx context.Context, s *server, raw jsontext.Value) (string, return "hibernated (the next call that reaches the guest wakes it transparently)", nil } -type promoteArgs struct { - sandboxArg - TemplateName string `json:"template_name"` -} - func toolPromote(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[promoteArgs](raw) if err != nil { @@ -420,6 +412,14 @@ func toolNodeInfo(ctx context.Context, s *server, _ jsontext.Value) (string, err return jsonText(info), nil } +func parseCommand(s *server, raw jsontext.Value) (cmdArgs, *sandbox.Sandbox, error) { + args, sb, err := s.parseAndBox[cmdArgs](raw) + if err == nil && args.Command == "" { + return args, nil, errors.New("command must not be empty") + } + return args, sb, err +} + func parse(raw jsontext.Value, v any) error { if len(raw) == 0 { return nil diff --git a/os-image/android/README.md b/os-image/android/README.md index 3957b54af..00718840b 100644 --- a/os-image/android/README.md +++ b/os-image/android/README.md @@ -3,9 +3,8 @@ Android guest for UI-automation sandboxes: the cocoon project's official redroid image (`ghcr.io/cocoonstack/cocoon/android:15.0`, upstream redroid 15 rootfs + cocoon boot chain, built from `cocoon/os-image/android/`) -plus silkd, claimable like any other template. Remote access is adb on -port 5555 — no VNC in this lineage. The intended pool shape is -`net: none`, `size: xlarge` (4 CPU / 8G). +plus silkd. Claim shape, adb access and behavior are in +[docs/android.md](../../docs/android.md); this file holds the build notes. ## Build @@ -30,14 +29,3 @@ no NIC, 4 CPU / 8G). Do not revert to the previous third-party VNC android pin: that build ships a broken dexpreopt boot-image chain that crash-loops zygote before the framework ever completes. - -## Constraints - -- **Boot chain**: the image boots its embedded 6.8 kernel via Android - `/init`, not the sandbox boot chain (PVH kernel + overlay-root init). - Use Cloud Hypervisor with `net: none`; snapshot/restore of a booted Android - is covered by the `androidsmoke` acceptance round. -- **silkd on Android**: the musl-static binary runs against an Android - userspace (`/system/bin/sh`, no `/bin/sh`); exec behavior is covered by - the `androidsmoke` acceptance round, and the session verbs need bash, which - the image lacks. diff --git a/os-image/browser/README.md b/os-image/browser/README.md index 244d75517..4fecd6399 100644 --- a/os-image/browser/README.md +++ b/os-image/browser/README.md @@ -1,12 +1,8 @@ # browser flavor -Headless-browser guest for CDP automation sandboxes: `base:24.04` plus a -pinned [Chrome for Testing](https://googlechromelabs.github.io/chrome-for-testing/) -build launched at boot with its DevTools (CDP) endpoint on guest loopback -`9222`. An agent claims it and connects any CDP client (Playwright -`connectOverCDP`, Puppeteer) through the silkd port relay. The intended -pool shape is `size: large` (4 CPU / 4G) — a persistent Chromium idles at -~150–250 MB RSS. +Headless-browser guest: `base:24.04` plus a pinned Chrome for Testing build +launched at boot. Claim shape, CDP access and behavior are in +[docs/browser.md](../../docs/browser.md); this file holds the build notes. ## Build @@ -25,14 +21,3 @@ with machine-readable pinned download URLs per version. Version bumps: update `CHROME_VERSION` + `CHROME_SHA256` from `last-known-good-versions-with-downloads.json`, same operational load as `COCOON_AGENT_VERSION`. - -## CDP on loopback is intentional - -Chromium M113+ binds `--remote-debugging-port` to `127.0.0.1` only. In -this stack that is exactly right: silkd's forwarder dials guest loopback, -so CDP is fully reachable through an authorized claim over vsock and never -exposed on a guest NIC — including on the no-network lane. - -`chromium.service` is a product service, not a readiness gate: the claim -still returns the moment silkd answers; Chromium comes up a beat later and -is polled by the workload (`e2e/cmd/browsersmoke` shows the pattern). diff --git a/protocol/README.md b/protocol/README.md index aa7d4a02e..b3a55aabf 100644 --- a/protocol/README.md +++ b/protocol/README.md @@ -5,10 +5,12 @@ implementations of the silkd protocol: - silkd (Rust, guest) parses/emits these in `silkd/src/proto.rs` tests. - `protocol/wire` (Go, host) round-trips the corpus in its tests; it is the one Go implementation, consumed by both the Go SDK and sandboxd. -- the Python SDK round-trips it in `sdk/python/tests/test_fixtures.py`. +- the Python SDK round-trips the request frames and checks only that each + response decodes to the fixture's `type` tag (and `data`), in + `sdk/python/tests/test_fixtures.py`. `req_*.json` are client→server frames, `resp_*.json` server→client. A frame -that only one side can round-trip is a protocol drift bug. The Rust side also +that the Rust or Go side cannot round-trip is a protocol drift bug. The Rust side also pins the corpus size, so adding a verb without its fixture fails CI. `enums.json` pins each wire enum's full value set (error/event/file kinds, diff --git a/protocol/wire/frame.go b/protocol/wire/frame.go index 53fb3e561..d9c5defed 100644 --- a/protocol/wire/frame.go +++ b/protocol/wire/frame.go @@ -97,7 +97,7 @@ var ( } // responseDecoders dispatches on the type tag first: one key differs in shape across variants. - responseDecoders = map[string]respDecoder{ + responseDecoders = map[string]func([]byte) (Response, error){ "started": decodeResp[Started], "stdout": decodeResp[Stdout], "stderr": decodeResp[Stderr], @@ -628,8 +628,6 @@ type GitBranches struct { func (GitBranches) RespType() string { return "git_branches" } -type respDecoder func([]byte) (Response, error) - type bulkResponse struct { head []byte mk func([]byte) Response @@ -759,7 +757,7 @@ func decodeBulk(line, head []byte, scratch *[]byte) ([]byte, bool) { return out[:n], true } -// encodeTagged splices the tag head in front of v's fields, so wire tags never live on the structs; the spare capacity byte lets Conn.Send append the newline without a copy. +// encodeTagged splices the tag head in front of v's fields, so wire tags never live on the structs. func encodeTagged(head string, v any) ([]byte, error) { body, err := json.Marshal(v) if err != nil { diff --git a/sandboxd/archives/pg/pg.go b/sandboxd/archives/pg/pg.go index 33afe230f..15b37bcc4 100644 --- a/sandboxd/archives/pg/pg.go +++ b/sandboxd/archives/pg/pg.go @@ -4,7 +4,6 @@ package pg import ( "context" _ "embed" - "errors" "time" "github.com/jackc/pgx/v5" @@ -91,7 +90,7 @@ func (r *Registry) TakeOwned(ctx context.Context, owner string, expect map[strin for id, epoch := range expect { ids, epochs = append(ids, id), append(epochs, epoch) } - recs, err := r.collect(ctx, `UPDATE sandboxd_archives a SET epoch = nextval('sandboxd_archives_epoch'), updated_at = now() + recs, err := cas.All(ctx, r.pool, scan, `UPDATE sandboxd_archives a SET epoch = nextval('sandboxd_archives_epoch'), updated_at = now() FROM unnest($2::text[], $3::bigint[]) AS t(id, epoch) WHERE a.cell = $1 AND a.id = t.id AND a.epoch = t.epoch AND a.owner = $4 AND NOT a.consumed RETURNING a.id, a.tenant, a.token_sha256, a.owner, a.epoch, a.consumed, a.checkpoint, a.deadline, NULL::jsonb`, r.cell, ids, epochs, owner) @@ -116,26 +115,17 @@ func (r *Registry) Delete(ctx context.Context, id string, epoch int64) error { } func (r *Registry) Get(ctx context.Context, id string) (archives.Record, bool, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - rec, err := scan(r.pool.QueryRow(qctx, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, claim - FROM sandboxd_archives WHERE cell = $2 AND id = $1`, id, r.cell)) - switch { - case errors.Is(err, pgx.ErrNoRows): - return archives.Record{}, false, nil - case err != nil: - return archives.Record{}, false, cas.Down(err) - } - return rec, true, nil + return cas.One(ctx, r.pool, scan, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, claim + FROM sandboxd_archives WHERE cell = $2 AND id = $1`, id, r.cell) } func (r *Registry) Sweepable(ctx context.Context, now time.Time, owner string) ([]archives.Record, error) { - return r.collect(ctx, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, NULL::jsonb + return cas.All(ctx, r.pool, scan, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, NULL::jsonb FROM sandboxd_archives WHERE cell = $2 AND (consumed OR owner = $3 OR deadline < $1 AND `+leaseExpired+`)`, now, r.cell, owner) } func (r *Registry) Rows(ctx context.Context, ids []string) (map[string]archives.Record, error) { - recs, err := r.collect(ctx, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, NULL::jsonb + recs, err := cas.All(ctx, r.pool, scan, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, NULL::jsonb FROM sandboxd_archives WHERE cell = $1 AND id = ANY($2)`, r.cell, ids) if err != nil { return nil, err @@ -164,20 +154,6 @@ func (r *Registry) Close() { r.pool.Close() } -func (r *Registry) collect(ctx context.Context, sql string, args ...any) ([]archives.Record, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - rows, err := r.pool.Query(qctx, sql, args...) - if err != nil { - return nil, cas.Down(err) - } - recs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (archives.Record, error) { return scan(row) }) - if err != nil { - return nil, cas.Down(err) - } - return recs, nil -} - func scan(row pgx.Row) (archives.Record, error) { var ( rec archives.Record diff --git a/sandboxd/ca.go b/sandboxd/ca.go index 64df934d7..3d388bcf1 100644 --- a/sandboxd/ca.go +++ b/sandboxd/ca.go @@ -107,9 +107,6 @@ func writeKeyMaterial(path string, data []byte, perm os.FileMode, force bool) er flags |= os.O_EXCL } f, err := os.OpenFile(path, flags, perm) //nolint:gosec // operator-chosen -out path - if errors.Is(err, os.ErrExist) { - return fmt.Errorf("%s exists; refusing to overwrite (use -force)", path) - } if err != nil { return err } diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 36a4a78f6..64c14c220 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -835,8 +835,8 @@ func (c *Config) validateMemoryLimit() error { func (c *Config) validateMetaStore() error { ms := c.MetaStore switch { - case c.ArchiveFailover && (ms == nil || ms.Cell == "" || !c.S3Checkpoints() || c.Mesh == nil): - return fmt.Errorf("archive_failover requires meta_store with a cell, an s3 checkpoint_store and a mesh") + case c.ArchiveFailover && (ms == nil || ms.Cell == "" || !c.S3Checkpoints() || c.Mesh == nil || c.Mesh.ClusterKey == ""): + return fmt.Errorf("archive_failover requires meta_store with a cell, an s3 checkpoint_store and a mesh with a cluster_key") case ms == nil: return nil case ms.Kind != "pg": @@ -954,7 +954,7 @@ func validateShards(names []string, field, kind string) error { return nil } -// namesHost reads addr the way Mint does and reports whether it carries a host a client could dial. +// namesHost reads addr the way NewPreviewServer does and reports whether it carries a host a client could dial. func namesHost(addr string) bool { if !strings.Contains(addr, "://") { addr = "http://" + addr diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 07284879a..37c2e3f5c 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -41,10 +41,20 @@ func TestLoadAppliesDefaults(t *testing.T) { } } -func TestLoadReadsCaptureTrim(t *testing.T) { - cfg := mustLoad(t, writeConfig(t, `{"pools":[{"template":"rt:24.04","capture_trim":true},{"template":"py:3.12"}]}`)) - if !cfg.Pools[0].CaptureTrim || cfg.Pools[1].CaptureTrim { - t.Errorf("capture_trim %v/%v, want on for the pool that names it and off by default", cfg.Pools[0].CaptureTrim, cfg.Pools[1].CaptureTrim) +func TestLoadReadsPoolFlags(t *testing.T) { + for _, tt := range []struct { + key string + get func(PoolSpec) bool + }{ + {"capture_trim", func(p PoolSpec) bool { return p.CaptureTrim }}, + {"reader", func(p PoolSpec) bool { return p.Reader }}, + } { + t.Run(tt.key, func(t *testing.T) { + cfg := mustLoad(t, writeConfig(t, `{"pools":[{"template":"rt:24.04","`+tt.key+`":true},{"template":"py:3.12"}]}`)) + if got, other := tt.get(cfg.Pools[0]), tt.get(cfg.Pools[1]); !got || other { + t.Errorf("%s %v/%v, want on for the pool that names it and off by default", tt.key, got, other) + } + }) } } @@ -214,7 +224,7 @@ func TestLoadArchiveFailover(t *testing.T) { parts := map[string]string{ "meta_store": `"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"}`, "checkpoint_store": `"checkpoint_store":{"kind":"s3","s3":{"bucket":"b"}}`, - "mesh": `"mesh":{"bind":"node1:7946"}`, + "mesh": `"mesh":{"bind":"node1:7946","cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="}`, } body := func(skip string) string { b := `{"api_token":"r","advertise_addr":"node1:7777","archive_failover":true,"pools":[]` @@ -240,11 +250,14 @@ func TestLoadArchiveFailover(t *testing.T) { if _, err := Load(writeConfig(t, strings.Replace(body(""), `,"cell":"c1"`, "", 1))); err == nil || !strings.Contains(err.Error(), "archive_failover requires") { t.Errorf("archive_failover without meta_store.cell: %v, want a refusal", err) } + if _, err := Load(writeConfig(t, strings.Replace(body(""), `,"cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="`, "", 1))); err == nil || !strings.Contains(err.Error(), "archive_failover requires") { + t.Errorf("archive_failover without mesh.cluster_key: %v, want a refusal", err) + } } func TestLoadReplicateSeconds(t *testing.T) { t.Setenv("SANDBOX_TEST_DSN", "postgres://sandboxd@db/sandboxd") - failover := `"archive_failover":true,"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"},"checkpoint_store":{"kind":"s3","s3":{"bucket":"b"}},"mesh":{"bind":"node1:7946"},` + failover := `"archive_failover":true,"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"},"checkpoint_store":{"kind":"s3","s3":{"bucket":"b"}},"mesh":{"bind":"node1:7946","cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="},` body := func(on, pool string) string { return `{"api_token":"r","advertise_addr":"node1:7777",` + on + `"pools":[` + pool + `]}` } @@ -375,13 +388,6 @@ func TestLoadAcceptsPreviewAdvertiseShapes(t *testing.T) { } } -func TestLoadReadsReader(t *testing.T) { - cfg := mustLoad(t, writeConfig(t, `{"pools":[{"template":"rt:24.04","reader":true},{"template":"py:3.12"}]}`)) - if !cfg.Pools[0].Reader || cfg.Pools[1].Reader { - t.Errorf("reader %v/%v, want on for the pool that names it and off by default", cfg.Pools[0].Reader, cfg.Pools[1].Reader) - } -} - func TestLoadAcceptsVolumes(t *testing.T) { path := writeConfig(t, `{"pools":[],"volume_roots":["/srv/models/"],"volumes":[ {"name":"imagenet","path":"/srv/datasets/imagenet.img"}, diff --git a/sandboxd/egress/bench_test.go b/sandboxd/egress/bench_test.go index 5f6d1ee75..0354fe8ae 100644 --- a/sandboxd/egress/bench_test.go +++ b/sandboxd/egress/bench_test.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strings" "testing" ) @@ -146,11 +147,7 @@ func BenchmarkScrubBody(b *testing.B) { }{ {"64KiB-page", func() io.Reader { return bytes.NewReader(page) }}, {"sse-256-events", func() io.Reader { - events := make([]string, 256) - for i := range events { - events[i] = string(event) - } - return chunked(events...) + return chunked(slices.Repeat([]string{string(event)}, 256)...) }}, } { b.Run(arm.name+"/plain", func(b *testing.B) { @@ -178,14 +175,11 @@ func benchFront(b *testing.B, mode InterceptMode) (proxyAddr string, roots *x509 b.Cleanup(upstream.Close) rule := Rule{Host: "example.com", Intercept: mode} var ca *CA - roots = x509.NewCertPool() if mode == InterceptAlways { ca = testCAOnly(b) - if !roots.AppendCertsFromPEM(ca.CertPEM()) { - b.Fatal("append cluster root") - } + roots = rootsFor(b, ca) } else { - roots.AddCert(upstream.Certificate()) + roots = trustUpstream(upstream) } p := New(Policy{Allow: []Rule{rule}}, nil, ca, fixedDial(upstream.Listener.Addr().String()), nil, nil) if ca != nil { diff --git a/sandboxd/egress/ca_test.go b/sandboxd/egress/ca_test.go index af78f34cb..096a0873a 100644 --- a/sandboxd/egress/ca_test.go +++ b/sandboxd/egress/ca_test.go @@ -13,28 +13,16 @@ import ( ) func TestLoadCARejectsForeignIntermediate(t *testing.T) { - rootA, keyA, err := GenerateRoot("A") - if err != nil { - t.Fatalf("root A: %v", err) - } - rootB, _, err := GenerateRoot("B") - if err != nil { - t.Fatalf("root B: %v", err) - } - interCert, interKey, err := IssueIntermediate(rootA, keyA, "n") - if err != nil { - t.Fatalf("intermediate: %v", err) - } + rootA, keyA := testRoot(t, "A") + rootB, _ := testRoot(t, "B") + interCert, interKey := testIntermediate(t, rootA, keyA, "n") if _, err := LoadCA(rootB, interCert, interKey); err == nil { t.Error("LoadCA accepted an intermediate not signed by the given root") } } func TestLoadCARejectsRootAsIntermediate(t *testing.T) { - rootCert, rootKey, err := GenerateRoot("root") - if err != nil { - t.Fatalf("root: %v", err) - } + rootCert, rootKey := testRoot(t, "root") if _, err := LoadCA(rootCert, rootCert, rootKey); err == nil { t.Error("LoadCA accepted the root as its own intermediate; want rejection (would put the root key on the node)") } @@ -77,10 +65,7 @@ func TestSignLeafChainsToRoot(t *testing.T) { } func TestLoadCARejectsExpiredIntermediate(t *testing.T) { - rootCert, rootKey, err := GenerateRoot("root") - if err != nil { - t.Fatalf("root: %v", err) - } + rootCert, rootKey := testRoot(t, "root") root, err := parseCert(rootCert, "root") if err != nil { t.Fatalf("parse root: %v", err) @@ -123,18 +108,9 @@ func TestLoadCARejectsExpiredIntermediate(t *testing.T) { } func TestLoadCARootBundle(t *testing.T) { - rootA, keyA, err := GenerateRoot("A") - if err != nil { - t.Fatalf("root A: %v", err) - } - rootB, _, err := GenerateRoot("B") - if err != nil { - t.Fatalf("root B: %v", err) - } - interCert, interKey, err := IssueIntermediate(rootA, keyA, "n") - if err != nil { - t.Fatalf("intermediate: %v", err) - } + rootA, keyA := testRoot(t, "A") + rootB, _ := testRoot(t, "B") + interCert, interKey := testIntermediate(t, rootA, keyA, "n") bundle := slices.Concat(rootA, rootB) ca, err := LoadCA(bundle, interCert, interKey) if err != nil { @@ -153,14 +129,8 @@ func TestLoadCARootBundle(t *testing.T) { } func TestLoadCARejectsNonCertBlockInRoot(t *testing.T) { - rootCert, rootKey, err := GenerateRoot("root") - if err != nil { - t.Fatalf("root: %v", err) - } - interCert, interKey, err := IssueIntermediate(rootCert, rootKey, "n") - if err != nil { - t.Fatalf("intermediate: %v", err) - } + rootCert, rootKey := testRoot(t, "root") + interCert, interKey := testIntermediate(t, rootCert, rootKey, "n") polluted := slices.Concat(rootCert, rootKey) if _, err := LoadCA(polluted, interCert, interKey); err == nil { t.Error("LoadCA accepted a non-certificate pem block in the root bundle") @@ -168,12 +138,9 @@ func TestLoadCARejectsNonCertBlockInRoot(t *testing.T) { } func TestFingerprintIsRootAndStable(t *testing.T) { - rootCert, rootKey, err := GenerateRoot("root") - if err != nil { - t.Fatalf("root: %v", err) - } - c1, k1, _ := IssueIntermediate(rootCert, rootKey, "n1") - c2, k2, _ := IssueIntermediate(rootCert, rootKey, "n2") + rootCert, rootKey := testRoot(t, "root") + c1, k1 := testIntermediate(t, rootCert, rootKey, "n1") + c2, k2 := testIntermediate(t, rootCert, rootKey, "n2") ca1, err := LoadCA(rootCert, c1, k1) if err != nil { t.Fatalf("load ca1: %v", err) @@ -187,16 +154,28 @@ func TestFingerprintIsRootAndStable(t *testing.T) { } } -func testCA(tb testing.TB) (*CA, []byte) { +func testRoot(tb testing.TB, name string) (cert, key []byte) { tb.Helper() - rootCert, rootKey, err := GenerateRoot("test root") + cert, key, err := GenerateRoot(name) if err != nil { - tb.Fatalf("generate root: %v", err) + tb.Fatalf("generate root %s: %v", name, err) } - interCert, interKey, err := IssueIntermediate(rootCert, rootKey, "node1") + return cert, key +} + +func testIntermediate(tb testing.TB, rootCert, rootKey []byte, name string) (cert, key []byte) { + tb.Helper() + cert, key, err := IssueIntermediate(rootCert, rootKey, name) if err != nil { - tb.Fatalf("issue intermediate: %v", err) + tb.Fatalf("issue intermediate %s: %v", name, err) } + return cert, key +} + +func testCA(tb testing.TB) (*CA, []byte) { + tb.Helper() + rootCert, rootKey := testRoot(tb, "test root") + interCert, interKey := testIntermediate(tb, rootCert, rootKey, "node1") ca, err := LoadCA(rootCert, interCert, interKey) if err != nil { tb.Fatalf("load ca: %v", err) diff --git a/sandboxd/egress/intercept_test.go b/sandboxd/egress/intercept_test.go index 547825d73..0b50fcce1 100644 --- a/sandboxd/egress/intercept_test.go +++ b/sandboxd/egress/intercept_test.go @@ -540,8 +540,7 @@ func TestInterceptStreamsAnEventStreamAsItArrives(t *testing.T) { defer close(release) ca, _ := testCA(t) p := New(Policy{Allow: []Rule{{Host: "example.com", Intercept: InterceptAlways}}}, nil, ca, fixedDial(upstream.Listener.Addr().String()), nil, nil) - guestRoots := x509.NewCertPool() - guestRoots.AppendCertsFromPEM(ca.CertPEM()) + guestRoots := rootsFor(t, ca) front := serveTrusting(t, p, upstream) tc := connectTLS(t, front.Listener.Addr().String(), "example.com:443", "example.com", guestRoots) @@ -568,8 +567,7 @@ func TestAnUpstreamDyingMidBodyTruncatesTheInterceptedResponse(t *testing.T) { defer upstream.Close() ca, _ := testCA(t) p := New(Policy{Allow: []Rule{{Host: "example.com", Intercept: InterceptAlways}}}, tt.secrets, ca, fixedDial(upstream.Listener.Addr().String()), nil, nil) - guestRoots := x509.NewCertPool() - guestRoots.AppendCertsFromPEM(ca.CertPEM()) + guestRoots := rootsFor(t, ca) tc := connectTLS(t, serveTrusting(t, p, upstream).Listener.Addr().String(), "example.com:443", "example.com", guestRoots) defer func() { _ = tc.Close() }() req := newRequest(t, http.MethodGet, tt.target, "") @@ -703,11 +701,16 @@ func interceptProxySecrets(t *testing.T, policy Policy, secrets Secrets, events } p := New(policy, secrets, ca, fixedDial(upstream.Listener.Addr().String()), audit, nil) - guestRoots := x509.NewCertPool() - if !guestRoots.AppendCertsFromPEM(ca.CertPEM()) { - t.Fatal("guest trust: append ca cert") + return p, rootsFor(t, ca), upstream +} + +func rootsFor(tb testing.TB, ca *CA) *x509.CertPool { + tb.Helper() + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(ca.CertPEM()) { + tb.Fatal("guest trust: append ca cert") } - return p, guestRoots, upstream + return roots } func serveTrusting(t *testing.T, p *Proxy, upstream *httptest.Server) *httptest.Server { diff --git a/sandboxd/egress/policy.go b/sandboxd/egress/policy.go index db154fd9c..3a740eec7 100644 --- a/sandboxd/egress/policy.go +++ b/sandboxd/egress/policy.go @@ -238,39 +238,25 @@ func Compose(pool, tenant Policy) PolicyEvaluator { func (c composite) Eval(host, method string, port uint16) (Rule, Decision) { rule, pd := c.pool.Eval(host, method, port) - if pd == DecisionAllow { - if _, td := c.tenant.Eval(host, method, port); td == DecisionAllow { - return rule, DecisionAllow - } - } - return Rule{}, DecisionDeny + _, td := c.tenant.Eval(host, method, port) + return intersect(rule, pd, td) } func (c composite) EvalHost(host string, port uint16) (Rule, Decision) { rule, pd := c.pool.EvalHost(host, port) - if pd == DecisionAllow { - if _, td := c.tenant.EvalHost(host, port); td == DecisionAllow { - return rule, DecisionAllow - } - } - return Rule{}, DecisionDeny + _, td := c.tenant.EvalHost(host, port) + return intersect(rule, pd, td) } func (c composite) EvalInner(host, method string, port uint16) (Rule, Decision) { rule, pd := c.pool.EvalInner(host, method, port) - if pd == DecisionAllow { - if _, td := c.tenant.Eval(host, method, port); td == DecisionAllow { - return rule, DecisionAllow - } - } - return Rule{}, DecisionDeny + _, td := c.tenant.Eval(host, method, port) + return intersect(rule, pd, td) } func (c composite) EvalFlow(port uint16) Decision { - if c.pool.EvalFlow(port) == DecisionAllow && c.tenant.EvalFlow(port) == DecisionAllow { - return DecisionAllow - } - return DecisionDeny + _, d := intersect(Rule{}, c.pool.EvalFlow(port), c.tenant.EvalFlow(port)) + return d } // ServesSocks is the pool's call: the tenant's rules already gate every tunnel through Eval. @@ -285,3 +271,10 @@ func (c composite) ServesDNS() bool { func (c composite) InterceptsHost(pattern string) bool { return c.pool.InterceptsHost(pattern) && slices.ContainsFunc(c.tenant.Allow, func(r Rule) bool { return r.Covers(pattern) }) } + +func intersect(rule Rule, pd, td Decision) (Rule, Decision) { + if pd == DecisionAllow && td == DecisionAllow { + return rule, DecisionAllow + } + return Rule{}, DecisionDeny +} diff --git a/sandboxd/egress/proxy.go b/sandboxd/egress/proxy.go index 928cc859b..9e4666cc3 100644 --- a/sandboxd/egress/proxy.go +++ b/sandboxd/egress/proxy.go @@ -132,9 +132,6 @@ type connPools struct { routed map[routeKey]*http.Transport } -// DialFunc opens the upstream connection for a permitted request. -type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) - // Proxy is one sandbox's forward proxy, gated by Policy and audited per request. type Proxy struct { policy Evaluator diff --git a/sandboxd/egress/proxy_test.go b/sandboxd/egress/proxy_test.go index b0e539fd0..2064f5b6d 100644 --- a/sandboxd/egress/proxy_test.go +++ b/sandboxd/egress/proxy_test.go @@ -589,10 +589,11 @@ func TestARefusedDialAnswersItsStatusAndIsAuditedWithNoTransfer(t *testing.T) { if ev := recvEvent(t, events); ev.Decision != tt.decision { t.Errorf("CONNECT audit %+v, want decision %v", ev, tt.decision) } + front.Close() select { case ev := <-transfers: t.Errorf("a refused dial reported a transfer for %s", ev.Host) - case <-time.After(200 * time.Millisecond): + default: } }) } diff --git a/sandboxd/egress/secrets.go b/sandboxd/egress/secrets.go index 06fc3a465..087e72dd5 100644 --- a/sandboxd/egress/secrets.go +++ b/sandboxd/egress/secrets.go @@ -11,7 +11,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// nonInjectable lists the secret headers the transport owns or strips. var nonInjectable = slices.Concat([]string{"Host", "Content-Length"}, hopHeaders) // SecretSpec declares a credential the proxy injects, valued from the env ValueEnv names (a claim's host-only entry first, then the node's), or from the claim env Name when ValueEnv is empty. diff --git a/sandboxd/egress/socks_test.go b/sandboxd/egress/socks_test.go index c980cc384..3970816e5 100644 --- a/sandboxd/egress/socks_test.go +++ b/sandboxd/egress/socks_test.go @@ -97,14 +97,18 @@ func TestSocksRefusesWhatItCannotServe(t *testing.T) { t.Run(tt.name, func(t *testing.T) { events := make(chan Event, 4) _, addr := socksProxy(t, Policy{Allow: []Rule{{Host: "*"}}}, nil, fixedDial("127.0.0.1:1"), events, nil) - _, reply := socksExchange(t, addr, tt.greeting, tt.request) + conn, reply := socksExchange(t, addr, tt.greeting, tt.request) if string(reply[:2]) != string(tt.want) { t.Errorf("reply = %v, want prefix %v", reply, tt.want) } + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + if _, err := io.Copy(io.Discard, conn); errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("the proxy never hung up: %v", err) + } select { case ev := <-events: t.Errorf("refused handshake recorded %+v", ev) - case <-time.After(50 * time.Millisecond): + default: } }) } diff --git a/sandboxd/egress/upstream.go b/sandboxd/egress/upstream.go index 69841f07d..ea883a34d 100644 --- a/sandboxd/egress/upstream.go +++ b/sandboxd/egress/upstream.go @@ -27,6 +27,9 @@ type Router interface { Dial(ctx context.Context, route, network, addr string) (net.Conn, error) } +// DialFunc opens the upstream connection for a permitted request. +type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + // UpstreamAllow is the set of proxies a claim may route through: exact host names and IP prefixes. type UpstreamAllow struct { hosts []string diff --git a/sandboxd/engine/cloneargs_test.go b/sandboxd/engine/cloneargs_test.go index 7ea2a8078..24a790b35 100644 --- a/sandboxd/engine/cloneargs_test.go +++ b/sandboxd/engine/cloneargs_test.go @@ -93,16 +93,6 @@ func TestEgressVMsSpreadOverEveryConfiguredShard(t *testing.T) { } } -func TestNetworkChoiceIsStableForAName(t *testing.T) { - shards := []string{"a", "b", "c"} - first := shardOf(shards, "sbx-pool1-42") - for range 100 { - if got := shardOf(shards, "sbx-pool1-42"); got != first { - t.Fatalf("shardOf returned %q then %q for one name", first, got) - } - } -} - func TestNetArgsHonorsTheLaneAndTheAttachment(t *testing.T) { none := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeMedium} egress := types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeMedium} diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 95873c923..5bbd0292f 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -14,6 +14,7 @@ import ( "os" "os/exec" "runtime" + "slices" "strconv" "strings" "sync" @@ -390,7 +391,7 @@ func (e *Engine) probe(ctx context.Context, vsockSocket string, port int, timeou } func (e *Engine) cloneArgs(name string, key types.PoolKey, source ...string) []string { - return append(append([]string{"vm", "clone"}, source...), e.cloneFlags(name, key, e.restoreArgs())...) + return slices.Concat([]string{"vm", "clone"}, source, e.cloneFlags(name, key, e.restoreArgs())) } func (e *Engine) cloneFlags(name string, key types.PoolKey, restore []string) []string { @@ -401,8 +402,7 @@ func (e *Engine) cloneFlags(name string, key types.PoolKey, restore []string) [] } func (e *Engine) restoreCmdArgs(vmName, snapRef string) []string { - args := append([]string{"vm", "restore", argOutput, formatJSON}, e.restoreArgs()...) - return append(args, vmName, snapRef) + return slices.Concat([]string{"vm", "restore", argOutput, formatJSON}, e.restoreArgs(), []string{vmName, snapRef}) } func (e *Engine) directIOArg() string { diff --git a/sandboxd/engine/engine_test.go b/sandboxd/engine/engine_test.go index a9b39d713..8a81707b9 100644 --- a/sandboxd/engine/engine_test.go +++ b/sandboxd/engine/engine_test.go @@ -105,20 +105,6 @@ func TestProbeRetriesPastFailures(t *testing.T) { } } -func TestProbeRetriesUntilListenerAppears(t *testing.T) { - path := sockPath(t) - done := make(chan error, 1) - go func() { - done <- newTestEngine().Probe(t.Context(), path, 2*time.Second) - }() - - time.Sleep(60 * time.Millisecond) - listenMuxer(t, path, "OK 2048\n", infoFrame) - if err := <-done; err != nil { - t.Errorf("Probe: %v", err) - } -} - func TestDialGuestPortCtxCancel(t *testing.T) { path := sockPath(t) ln, err := net.Listen("unix", path) diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index 84e6f945d..d5a1a18f5 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -40,7 +40,7 @@ func TestInstallCACertWritesCertAndUpdates(t *testing.T) { func TestInstallCACertNonzeroExitFails(t *testing.T) { path := sockPath(t) fake := serveFakeSilkd(t, path) - fake.execCode = 3 + fake.set(func(f *fakeSilkd) { f.execCode = 3 }) err := newTestEngine().InstallCACert(t.Context(), path, []byte("x")) if err == nil || !strings.Contains(err.Error(), "exit code 3") { t.Errorf("got %v, want exit code 3 failure", err) @@ -50,7 +50,7 @@ func TestInstallCACertNonzeroExitFails(t *testing.T) { func TestInstallCACertWriteErrorFrameFails(t *testing.T) { path := sockPath(t) fake := serveFakeSilkd(t, path) - fake.writeErr = "disk full" + fake.set(func(f *fakeSilkd) { f.writeErr = "disk full" }) err := newTestEngine().InstallCACert(t.Context(), path, []byte("x")) if err == nil || !strings.Contains(err.Error(), "disk full") { t.Errorf("got %v, want fs_write error-frame failure", err) diff --git a/sandboxd/engine/portconn_test.go b/sandboxd/engine/portconn_test.go index 5801621ed..4d7b13707 100644 --- a/sandboxd/engine/portconn_test.go +++ b/sandboxd/engine/portconn_test.go @@ -17,7 +17,7 @@ import ( ) func TestGuestPortConnReadsDataUntilDone(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) go func() { frame, _ := json.Marshal(map[string]string{"type": "data", "data": base64.StdEncoding.EncodeToString([]byte("HTTP/1.1 200 OK"))}, json.Deterministic(true)) @@ -37,7 +37,7 @@ func TestGuestPortConnReadsDataUntilDone(t *testing.T) { func TestGuestPortConnCutBeforeDoneIsUnexpectedEOF(t *testing.T) { for name, trailer := range map[string]string{"between frames": "", "mid frame": `{"type":"da`} { t.Run(name, func(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) go func() { frame, _ := json.Marshal(map[string]string{"type": "data", "data": base64.StdEncoding.EncodeToString([]byte("partial"))}, json.Deterministic(true)) @@ -57,7 +57,7 @@ func TestGuestPortConnCutBeforeDoneIsUnexpectedEOF(t *testing.T) { } func TestGuestPortConnWriteWrapsInDataFrame(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) go func() { _, _ = gc.Write([]byte("GET / HTTP/1.1\r\n")) }() @@ -77,7 +77,7 @@ func TestGuestPortConnWriteWrapsInDataFrame(t *testing.T) { } func TestGuestPortConnCloseWriteSendsDataEnd(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) go func() { _ = gc.CloseWrite() }() @@ -105,7 +105,7 @@ func TestGuestPortConnRoundTripsFixtures(t *testing.T) { return append(bytes.TrimSpace(raw), '\n') } - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) respData, respDone := fixture("resp_data.json"), fixture("resp_done.json") go func() { @@ -121,18 +121,14 @@ func TestGuestPortConnRoundTripsFixtures(t *testing.T) { t.Errorf("fixture done frame: %v, want EOF", err) } - client2, silk2 := net.Pipe() - t.Cleanup(func() { _ = client2.Close(); _ = silk2.Close() }) - gc2 := &guestPortConn{Conn: client2, r: bufio.NewReaderSize(client2, portReadBuf)} + silk2, gc2 := newTestGuestPortConn(t, portReadBuf) respError := fixture("resp_error.json") go func() { _, _ = silk2.Write(respError) }() if _, err := gc2.Read(buf); err == nil || errors.Is(err, io.EOF) { t.Errorf("fixture error frame: %v, want a typed error", err) } - client3, silk3 := net.Pipe() - t.Cleanup(func() { _ = client3.Close(); _ = silk3.Close() }) - gc3 := &guestPortConn{Conn: client3, r: bufio.NewReaderSize(client3, portReadBuf)} + silk3, gc3 := newTestGuestPortConn(t, portReadBuf) go func() { _, _ = gc3.Write([]byte("hello")) }() line := make([]byte, 256) n, _ = silk3.Read(line) @@ -152,7 +148,7 @@ func TestPortReadBufHoldsOneBulkFrame(t *testing.T) { } func TestGuestPortConnReadsAFirstFrameThatIsAFullBulkFrame(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) payload := patterned(wire.BulkChunk) go writeDataFrames(t, silk, payload) @@ -163,7 +159,7 @@ func TestGuestPortConnReadsAFirstFrameThatIsAFullBulkFrame(t *testing.T) { } func TestGuestPortConnReadsSmallFramesThenBulkFrames(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) chunks := [][]byte{[]byte("HTTP/1.1 200 OK\r\n"), patterned(100), patterned(3000), patterned(wire.BulkChunk), patterned(wire.BulkChunk - 1), []byte("tail")} go writeDataFrames(t, silk, chunks...) @@ -179,7 +175,7 @@ func TestGuestPortConnReadsSmallFramesThenBulkFrames(t *testing.T) { func TestGuestPortConnRefusesAFrameOverMaxFrame(t *testing.T) { for name, trailer := range map[string]string{"unterminated": "", "terminated": "\n"} { t.Run(name, func(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) go func() { _, _ = silk.Write(append(bytes.Repeat([]byte("a"), wire.MaxFrame+1), trailer...)) @@ -192,11 +188,11 @@ func TestGuestPortConnRefusesAFrameOverMaxFrame(t *testing.T) { } } -func newTestGuestPortConn(t *testing.T) (net.Conn, *guestPortConn) { +func newTestGuestPortConn(t *testing.T, size int) (net.Conn, *guestPortConn) { t.Helper() client, silk := net.Pipe() t.Cleanup(func() { _ = client.Close(); _ = silk.Close() }) - return silk, &guestPortConn{Conn: client, r: bufio.NewReaderSize(client, portForwardMax)} + return silk, &guestPortConn{Conn: client, r: bufio.NewReaderSize(client, size)} } func writeDataFrames(t *testing.T, silk net.Conn, chunks ...[]byte) { diff --git a/sandboxd/engine/silkd.go b/sandboxd/engine/silkd.go index a5dc6e262..9b1e7c5ad 100644 --- a/sandboxd/engine/silkd.go +++ b/sandboxd/engine/silkd.go @@ -94,7 +94,7 @@ func (e *Engine) silkdStream(ctx context.Context, vsockSocket string, req wire.R case *wire.Done: return nil case *wire.ErrorResp: - return fmt.Errorf("silkd %w", resp) + return frameErr(resp) default: if err := onFrame(resp); err != nil { return err @@ -108,7 +108,7 @@ func (e *Engine) silkdCollect[F wire.Response, E any](ctx context.Context, vsock err := e.silkdStream(ctx, vsockSocket, req, func(frame wire.Response) error { resp, ok := frame.(F) if !ok { - return fmt.Errorf("unexpected silkd frame %q", frame.RespType()) + return frameErr(frame) } out = append(out, items(resp)...) if len(out) > limit { @@ -147,10 +147,7 @@ func (e *Engine) silkdOnce[T wire.Response](ctx context.Context, vsockSocket str if _, ok := frame.(T); ok { return nil } - if resp, ok := frame.(*wire.ErrorResp); ok { - return fmt.Errorf("silkd %w", resp) - } - return fmt.Errorf("unexpected silkd frame %q", frame.RespType()) + return frameErr(frame) } func (e *Engine) silkdWriteFile(ctx context.Context, vsockSocket, path string, mode uint32, data []byte) error { @@ -177,14 +174,10 @@ func (e *Engine) silkdWriteFile(ctx context.Context, vsockSocket, path string, m if err != nil { return errors.Join(sendErr, err) } - switch resp := frame.(type) { - case *wire.Done: + if _, ok := frame.(*wire.Done); ok { return nil - case *wire.ErrorResp: - return fmt.Errorf("silkd %w", resp) - default: - return fmt.Errorf("unexpected silkd frame %q", resp.RespType()) } + return frameErr(frame) } func (e *Engine) silkdExec(ctx context.Context, vsockSocket string, argv ...string) error { @@ -220,10 +213,8 @@ func (e *Engine) silkdOutput(ctx context.Context, vsockSocket string, argv ...st return nil, &guestExitError{code: resp.Code, output: string(bytes.TrimSpace(out))} } return stdout, nil - case *wire.ErrorResp: - return nil, fmt.Errorf("silkd %w", resp) default: - return nil, fmt.Errorf("unexpected silkd frame %q", resp.RespType()) + return nil, frameErr(resp) } } } @@ -234,3 +225,10 @@ func appendCapped(out, data []byte) []byte { } return out } + +func frameErr(frame wire.Response) error { + if resp, ok := frame.(*wire.ErrorResp); ok { + return fmt.Errorf("silkd %w", resp) + } + return fmt.Errorf("unexpected silkd frame %q", frame.RespType()) +} diff --git a/sandboxd/engine/volume_test.go b/sandboxd/engine/volume_test.go index e3783a5b6..ee40b25d2 100644 --- a/sandboxd/engine/volume_test.go +++ b/sandboxd/engine/volume_test.go @@ -286,10 +286,10 @@ func TestMountVolumeReadsEachOtherSerialOnce(t *testing.T) { if fake.listCalls != 4 { t.Fatalf("list calls = %d, want 4 (3 blank serials + the successful poll)", fake.listCalls) } - if got := len(slices.DeleteFunc(slices.Clone(fake.readCalls), func(p string) bool { return p != "/sys/block/vda/serial" })); got != 1 { + if got := countOf(fake.readCalls, "/sys/block/vda/serial"); got != 1 { t.Errorf("read the root disk's serial %d times, want once: %v", got, fake.readCalls) } - if got := len(slices.DeleteFunc(slices.Clone(fake.readCalls), func(p string) bool { return p != "/sys/block/vdc/serial" })); got != 4 { + if got := countOf(fake.readCalls, "/sys/block/vdc/serial"); got != 4 { t.Errorf("read the awaited disk's serial %d times, want 4: %v", got, fake.readCalls) } } @@ -399,6 +399,16 @@ func TestSilkdListAndReadAreBounded(t *testing.T) { } } +func countOf(items []string, want string) int { + n := 0 + for _, item := range items { + if item == want { + n++ + } + } + return n +} + func configureVolumeDevices(f *fakeSilkd) { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 4a23002e5..6b58fa41c 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -57,9 +57,7 @@ func New(ctx context.Context, cfg *memberlist.Config, self NodeState, secretKey cfg.Name = self.NodeID cfg.Delegate = (*delegate)(m) cfg.Events = (*eventDelegate)(m) - if len(secretKey) > 0 { - cfg.SecretKey = secretKey - } + cfg.SecretKey = secretKey ml, err := memberlist.Create(cfg) if err != nil { return nil, fmt.Errorf("create memberlist: %w", err) diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index fcbeac466..8511e2bbb 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -177,19 +177,17 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { } a.mesh.UpdateSelf(map[string]int{"kk": 4}, []string{"tpl-hash"}, []string{"dataset"}) - deadline := time.Now().Add(10 * time.Second) - for time.Now().Before(deadline) { + learned := eventually(func() bool { cands := b.mesh.Candidates("kk") owners := b.mesh.TemplateOwners("tpl-hash") volumeOwners := b.mesh.VolumeOwners([]string{"dataset"}) - if len(cands) == 1 && cands[0] == "node-a:7777" && + return len(cands) == 1 && cands[0] == "node-a:7777" && len(owners) == 1 && owners[0] == "node-a:7777" && - len(volumeOwners) == 1 && volumeOwners[0] == "node-a:7777" { - return - } - time.Sleep(100 * time.Millisecond) + len(volumeOwners) == 1 && volumeOwners[0] == "node-a:7777" + }) + if !learned { + t.Fatalf("node-b never learned node-a's state: view=%+v", b.mesh.Members()) } - t.Fatalf("node-b never learned node-a's state: view=%+v", b.mesh.Members()) } func TestARuntimeDigestChangeReachesThePeer(t *testing.T) { @@ -199,14 +197,9 @@ func TestARuntimeDigestChangeReachesThePeer(t *testing.T) { t.Fatalf("join: %v", err) } a.mesh.UpdateDigest("rotated") - deadline := time.Now().Add(10 * time.Second) - for time.Now().Before(deadline) { - if b.mesh.ConfigMismatches() == 1 { - return - } - time.Sleep(100 * time.Millisecond) + if !eventually(func() bool { return b.mesh.ConfigMismatches() == 1 }) { + t.Fatalf("node-b never saw node-a's new digest: view=%+v", b.mesh.Members()) } - t.Fatalf("node-b never saw node-a's new digest: view=%+v", b.mesh.Members()) } func TestVolumeOwnersRequireEveryNameAndExcludeSelf(t *testing.T) { @@ -275,6 +268,15 @@ func TestVolumeHoldersCountSelfAndPeers(t *testing.T) { } } +func eventually(cond func() bool) bool { + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + if cond() { + return true + } + } + return false +} + func mergeStates(t *testing.T, m *Mesh, states []NodeState) { t.Helper() for _, st := range states { diff --git a/sandboxd/metastore/cas.go b/sandboxd/metastore/cas.go index d43132145..4ead2f197 100644 --- a/sandboxd/metastore/cas.go +++ b/sandboxd/metastore/cas.go @@ -9,6 +9,8 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) +type ScanFunc[T any] func(pgx.Row) (T, error) + // CAS runs a registry's compare-and-set queries and answers with its sentinels. type CAS struct { Lost error @@ -44,6 +46,33 @@ func (c CAS) Delete(ctx context.Context, pool *pgxpool.Pool, sql string, args .. return nil } +func (c CAS) One[T any](ctx context.Context, pool *pgxpool.Pool, scan ScanFunc[T], sql string, args ...any) (T, bool, error) { + qctx, cancel := context.WithTimeout(ctx, QueryTimeout) + defer cancel() + v, err := scan(pool.QueryRow(qctx, sql, args...)) + switch { + case errors.Is(err, pgx.ErrNoRows): + return *new(T), false, nil + case err != nil: + return *new(T), false, c.Down(err) + } + return v, true, nil +} + +func (c CAS) All[T any](ctx context.Context, pool *pgxpool.Pool, scan ScanFunc[T], sql string, args ...any) ([]T, error) { + qctx, cancel := context.WithTimeout(ctx, QueryTimeout) + defer cancel() + rows, err := pool.Query(qctx, sql, args...) + if err != nil { + return nil, c.Down(err) + } + out, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (T, error) { return scan(row) }) + if err != nil { + return nil, c.Down(err) + } + return out, nil +} + // Down wraps err in Unavailable. func (c CAS) Down(err error) error { return fmt.Errorf("%w: %w", c.Unavailable, err) diff --git a/sandboxd/metastore/metastoretest/metastoretest.go b/sandboxd/metastore/metastoretest/metastoretest.go index e4166739e..5726a7033 100644 --- a/sandboxd/metastore/metastoretest/metastoretest.go +++ b/sandboxd/metastore/metastoretest/metastoretest.go @@ -4,7 +4,6 @@ package metastoretest import ( "context" "crypto/rand" - "encoding/hex" "os" "strings" "testing" @@ -19,9 +18,7 @@ func PGSchema(t *testing.T) string { if url == "" { t.Skip("SANDBOXD_TEST_PG_URL unset") } - raw := make([]byte, 6) - _, _ = rand.Read(raw) - name := "t_" + hex.EncodeToString(raw) + name := "t_" + strings.ToLower(rand.Text()) exec := func(ctx context.Context, sql string) error { conn, err := pgx.Connect(ctx, url) if err != nil { diff --git a/sandboxd/moves/movestest/movestest.go b/sandboxd/moves/movestest/movestest.go index 46cb3c3f6..c16e28312 100644 --- a/sandboxd/moves/movestest/movestest.go +++ b/sandboxd/moves/movestest/movestest.go @@ -115,8 +115,8 @@ func (m *Mem) write(rec moves.Record) int64 { return rec.Epoch } -// Contract checks the behavior every backend shares. -func Contract(t *testing.T, open Opener) { +// Run checks the behavior every backend shares. +func Run(t *testing.T, open Opener) { t.Run("begin get", func(t *testing.T) { testBeginGet(t, open(t)) }) t.Run("every allowed advance", func(t *testing.T) { testAdvance(t, open(t)) }) t.Run("advance is a compare-and-set", func(t *testing.T) { testAdvanceFences(t, open(t)) }) diff --git a/sandboxd/moves/movestest/movestest_test.go b/sandboxd/moves/movestest/movestest_test.go index 9c1d74dfb..8d9977c16 100644 --- a/sandboxd/moves/movestest/movestest_test.go +++ b/sandboxd/moves/movestest/movestest_test.go @@ -7,5 +7,5 @@ import ( ) func TestMemContract(t *testing.T) { - Contract(t, func(*testing.T) moves.Registry { return NewMem() }) + Run(t, func(*testing.T) moves.Registry { return NewMem() }) } diff --git a/sandboxd/moves/pg/pg.go b/sandboxd/moves/pg/pg.go index beebc9b10..5ce7b9c9c 100644 --- a/sandboxd/moves/pg/pg.go +++ b/sandboxd/moves/pg/pg.go @@ -4,7 +4,6 @@ package pg import ( "context" _ "embed" - "errors" "time" "github.com/jackc/pgx/v5" @@ -59,17 +58,8 @@ func (r *Registry) Advance(ctx context.Context, id string, expect int64, from, t } func (r *Registry) Get(ctx context.Context, id string) (moves.Record, bool, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - rec, err := scan(r.pool.QueryRow(qctx, `SELECT id, from_node, to_node, state, epoch, updated_at - FROM sandboxd_moves WHERE cell = $1 AND id = $2`, r.cell, id)) - switch { - case errors.Is(err, pgx.ErrNoRows): - return moves.Record{}, false, nil - case err != nil: - return moves.Record{}, false, cas.Down(err) - } - return rec, true, nil + return cas.One(ctx, r.pool, scan, `SELECT id, from_node, to_node, state, epoch, updated_at + FROM sandboxd_moves WHERE cell = $1 AND id = $2`, r.cell, id) } func (r *Registry) Delete(ctx context.Context, id string, epoch int64) error { @@ -77,18 +67,8 @@ func (r *Registry) Delete(ctx context.Context, id string, epoch int64) error { } func (r *Registry) Stale(ctx context.Context, before time.Time) ([]moves.Record, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - rows, err := r.pool.Query(qctx, `SELECT id, from_node, to_node, state, epoch, updated_at + return cas.All(ctx, r.pool, scan, `SELECT id, from_node, to_node, state, epoch, updated_at FROM sandboxd_moves WHERE cell = $1 AND updated_at < $2`, r.cell, before) - if err != nil { - return nil, cas.Down(err) - } - recs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (moves.Record, error) { return scan(row) }) - if err != nil { - return nil, cas.Down(err) - } - return recs, nil } // Close releases the connection pool. diff --git a/sandboxd/moves/pg/pg_test.go b/sandboxd/moves/pg/pg_test.go index 528c2ba86..8c15d19e8 100644 --- a/sandboxd/moves/pg/pg_test.go +++ b/sandboxd/moves/pg/pg_test.go @@ -11,7 +11,7 @@ import ( ) func TestRegistryContract(t *testing.T) { - movestest.Contract(t, func(t *testing.T) moves.Registry { return open(t, metastoretest.PGSchema(t), "c1") }) + movestest.Run(t, func(t *testing.T) moves.Registry { return open(t, metastoretest.PGSchema(t), "c1") }) } func TestCellsKeepTheirOwnRecords(t *testing.T) { diff --git a/sandboxd/outbound/dial_test.go b/sandboxd/outbound/dial_test.go index df8abf9c1..3bca90661 100644 --- a/sandboxd/outbound/dial_test.go +++ b/sandboxd/outbound/dial_test.go @@ -1,6 +1,7 @@ package outbound import ( + "net" "net/netip" "strings" "testing" @@ -28,24 +29,18 @@ func TestEgressDialerBlocksInternal(t *testing.T) { "[64:ff9b::a9fe:a9fe]:80", "[64:ff9b::a00:1]:80", } + d := newDialer(verdictOf(nil)) for _, addr := range blocked { - if err := newDialer(verdictOf(nil)).Control("tcp", addr, nil); err == nil { - t.Errorf("dial to internal %s allowed; SSRF not blocked", addr) - } + assertControl(t, d, addr, false) } for _, addr := range []string{"93.184.216.34:443", "[2606:4700:4700::1111]:443", "[64:ff9b::5db8:d822]:443"} { - if err := newDialer(verdictOf(nil)).Control("tcp", addr, nil); err != nil { - t.Errorf("dial to public %s blocked: %v", addr, err) - } + assertControl(t, d, addr, true) } } func TestEgressDialerAdmitsOnlyNamedInternalPrefixes(t *testing.T) { d := newDialer(verdictOf([]string{"fdc8::/16", "10.8.0.0/16"})) - check := func(addr string) error { - return d.Control("tcp", addr, nil) - } for name, tt := range map[string]struct { addr string allowed bool @@ -63,13 +58,7 @@ func TestEgressDialerAdmitsOnlyNamedInternalPrefixes(t *testing.T) { "CGNAT metadata": {"100.100.100.200:80", false}, } { t.Run(name, func(t *testing.T) { - err := check(tt.addr) - if tt.allowed && err != nil { - t.Errorf("%s was blocked: %v", tt.addr, err) - } - if !tt.allowed && err == nil { - t.Errorf("%s was allowed; it must not be reachable from a guest", tt.addr) - } + assertControl(t, d, tt.addr, tt.allowed) }) } } @@ -84,9 +73,7 @@ func TestEgressDialerWildcardAllowsEverything(t *testing.T) { "127.0.0.1:7777", "169.254.169.254:80", } { - if err := d.Control("tcp", addr, nil); err != nil { - t.Errorf("%s blocked under a wildcard allow-list: %v", addr, err) - } + assertControl(t, d, addr, true) } } @@ -118,13 +105,7 @@ func TestEgressDialerScopesInternalAllowToPorts(t *testing.T) { "outside the prefix": {"10.8.0.2:18090", false}, } { t.Run(name, func(t *testing.T) { - err := d.Control("tcp", tt.addr, nil) - if tt.allowed && err != nil { - t.Errorf("%s was blocked: %v", tt.addr, err) - } - if !tt.allowed && (err == nil || !strings.Contains(err.Error(), "blocked internal address")) { - t.Errorf("%s: %v, want blocked internal address", tt.addr, err) - } + assertControl(t, d, tt.addr, tt.allowed) }) } } @@ -145,25 +126,11 @@ func TestEgressDialerBlocksTheNodesOwnAddresses(t *testing.T) { "another public host": {"93.184.216.35:22", true}, } { t.Run(name, func(t *testing.T) { - err := d.Control("tcp", tt.addr, nil) - if tt.allowed && err != nil { - t.Errorf("%s was blocked: %v", tt.addr, err) - } - if !tt.allowed && (err == nil || !strings.Contains(err.Error(), "blocked internal address")) { - t.Errorf("%s: %v, want blocked internal address", tt.addr, err) - } + assertControl(t, d, tt.addr, tt.allowed) }) } } -func TestSelfAddrsAreOnlyPublic(t *testing.T) { - for _, ip := range selfAddrs() { - if !ip.IsGlobalUnicast() || ip.IsPrivate() || ip.Is4In6() { - t.Errorf("self address %s is not a public unicast address", ip) - } - } -} - func TestTheHostGuardFollowsARefresh(t *testing.T) { h := newHost(t, &fakeEngine{}, &config.Config{}, nil) stale := netip.MustParseAddr("93.184.216.34") @@ -191,3 +158,14 @@ func verdictOf(allow []string, self ...netip.Addr) VerdictFunc { a := parseInternalAllow(allow) return func(ip netip.Addr, port uint16) (bool, error) { return destVerdict(a, self, ip, port) } } + +func assertControl(t *testing.T, d *net.Dialer, addr string, allowed bool) { + t.Helper() + err := d.Control("tcp", addr, nil) + if allowed && err != nil { + t.Errorf("%s was blocked: %v", addr, err) + } + if !allowed && (err == nil || !strings.Contains(err.Error(), "blocked internal address")) { + t.Errorf("%s: %v, want blocked internal address; it must not be reachable from a guest", addr, err) + } +} diff --git a/sandboxd/outbound/dns.go b/sandboxd/outbound/dns.go index c07f4c169..bd693f390 100644 --- a/sandboxd/outbound/dns.go +++ b/sandboxd/outbound/dns.go @@ -1,7 +1,6 @@ package outbound import ( - "bufio" "cmp" "context" "encoding/binary" @@ -101,10 +100,6 @@ func (r *resolver) serve(ln net.Listener, datagram bool) { } } -func (r *resolver) close() { - r.cancel() -} - func (r *resolver) serveConn(conn net.Conn, datagram bool) { defer func() { _ = conn.Close() }() stop := context.AfterFunc(r.ctx, func() { _ = conn.Close() }) @@ -413,15 +408,10 @@ func writeDNSMsg(w io.Writer, msg []byte) error { // readNameservers reads path's nameserver lines as host:53 addresses, the loopback default when it names none. func readNameservers(path string) []string { - f, err := os.Open(path) //nolint:gosec // the node's resolver config - if err != nil { - return []string{"127.0.0.1:53"} - } - defer func() { _ = f.Close() }() + data, _ := os.ReadFile(path) //nolint:gosec // the node's resolver config var out []string - sc := bufio.NewScanner(f) - for sc.Scan() { - fields := strings.Fields(sc.Text()) + for line := range strings.Lines(string(data)) { + fields := strings.Fields(line) if len(fields) >= 2 && fields[0] == "nameserver" { if ip, err := netip.ParseAddr(fields[1]); err == nil { out = append(out, netip.AddrPortFrom(ip, 53).String()) diff --git a/sandboxd/outbound/dns_test.go b/sandboxd/outbound/dns_test.go index 6446c032d..751aa99e7 100644 --- a/sandboxd/outbound/dns_test.go +++ b/sandboxd/outbound/dns_test.go @@ -3,6 +3,7 @@ package outbound import ( "bytes" "context" + "encoding/binary" "net" "net/netip" "os" @@ -68,7 +69,7 @@ func TestDNSDoorDropsInternalAnswersAndRemembersTheRest(t *testing.T) { cfg.EgressInternalAllow = []string{"10.9.0.0/16:5432"} events := &dnsEvents{} h := newHost(t, &fakeEngine{}, cfg, func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } o.RecordDNS = events.record }) h.self.Store(&[]netip.Addr{netip.MustParseAddr("93.184.216.99")}) @@ -103,7 +104,7 @@ func TestDNSDoorDropsInternalAnswersAndRemembersTheRest(t *testing.T) { func TestDNSDoorPassesAnUnfilteredAnswerByteForByte(t *testing.T) { upstream := newFakeNameserver(t) h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } }) conn := dialDNS(t, armDNS(t, h)) query := packQuery(t, dnsmessage.Header{ID: 0x5a5a, RecursionDesired: true}, "raw.test.") @@ -123,7 +124,7 @@ func TestDNSDoorDropsAnInternalGlueRecord(t *testing.T) { upstream := newFakeNameserver(t) events := &dnsEvents{} h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } o.RecordDNS = events.record }) msg := queryDoor(t, armDNS(t, h), "glue.test.") @@ -141,7 +142,7 @@ func TestDNSDoorDropsAnInternalGlueRecord(t *testing.T) { func TestDNSDoorDisarmEndsAnOpenConnection(t *testing.T) { upstream := newFakeNameserver(t) h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } }) sb := armDNS(t, h) conn := dialDNS(t, sb) @@ -159,7 +160,7 @@ func TestDNSDoorDisarmEndsAnOpenConnection(t *testing.T) { func TestDNSDoorsHandATruncatedAnswerToEachTransport(t *testing.T) { upstream := newFakeNameserver(t) h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } }) sb := armDNS(t, h) gram, err := net.Dial("unix", engine.DNSDatagramSocketPath(sb.VsockSocket)) @@ -179,7 +180,7 @@ func TestADNSDialBeforeTheClaimForcesARebind(t *testing.T) { upstream := newFakeNameserver(t) events := &dnsEvents{} h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } o.RecordDNS = events.record }) sb := outboundtest.VsockSandbox(t, "sb_dns", testKey) @@ -217,7 +218,7 @@ func TestDNSDoorMovesPastANameserverThatAnswersServfail(t *testing.T) { upstream := newFakeNameserver(t) failing := newServfailNameserver(t) h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{failing, upstream.udp} } + o.Nameservers = func() []string { return []string{failing, upstream} } }) if got := addresses(queryDoor(t, armDNS(t, h), "public.test.")); len(got) != 2 { t.Errorf("answers %v, want the second nameserver's two addresses", got) @@ -252,7 +253,7 @@ func TestDNSDoorRefusesAQueryFloodPastItsBucket(t *testing.T) { upstream := newFakeNameserver(t) events := &dnsEvents{} h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } o.RecordDNS = events.record }) sb := armDNS(t, h) @@ -279,7 +280,7 @@ func TestDNSDoorRefusesAQueryFloodPastItsBucket(t *testing.T) { func TestDNSDoorDropsAndCountsMalformedQueries(t *testing.T) { upstream := newFakeNameserver(t) h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } }) sb := armDNS(t, h) for _, tt := range []struct { @@ -324,7 +325,7 @@ func FuzzDNSDoor(f *testing.F) { var forwarded atomic.Int64 h := New(Options{View: func() *View { return v }, Nameservers: func() []string { forwarded.Add(1) - return []string{upstream.udp} + return []string{upstream} }}) valid := packQuery(f, dnsmessage.Header{RecursionDesired: true}, "public.test.") for _, seed := range [][]byte{ @@ -380,11 +381,7 @@ func TestReadNameserversReadsResolvConf(t *testing.T) { } } -type fakeNameserver struct { - udp string -} - -func newFakeNameserver(t testing.TB) *fakeNameserver { +func newFakeNameserver(t testing.TB) string { t.Helper() pc, ln := listenUDPAndTCP(t) t.Cleanup(func() { _ = pc.Close(); _ = ln.Close() }) @@ -412,7 +409,7 @@ func newFakeNameserver(t testing.TB) *fakeNameserver { }() } }() - return &fakeNameserver{udp: pc.LocalAddr().String()} + return pc.LocalAddr().String() } func listenUDPAndTCP(t testing.TB) (net.PacketConn, net.Listener) { @@ -616,7 +613,7 @@ func rawAnswer(q dnsmessage.Message) []byte { } func frame(msg []byte) []byte { - return append([]byte{byte(len(msg) >> 8), byte(len(msg))}, msg...) + return append(binary.BigEndian.AppendUint16(nil, uint16(len(msg))), msg...) } func packQuery(t testing.TB, hdr dnsmessage.Header, names ...string) []byte { @@ -651,7 +648,7 @@ func standardQuery(query []byte) bool { func selfPointerAdditional(query []byte) []byte { query[11] = 1 off := len(query) - return append(query, 0xc0|byte(off>>8), byte(off), 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 1, 2, 3, 4) + return append(binary.BigEndian.AppendUint16(query, 0xc000|uint16(off)), 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 1, 2, 3, 4) } type fuzzConn struct { diff --git a/sandboxd/outbound/doors.go b/sandboxd/outbound/doors.go index 3d28b0478..9afd52208 100644 --- a/sandboxd/outbound/doors.go +++ b/sandboxd/outbound/doors.go @@ -57,7 +57,7 @@ func (d *doors) close() { } } if d.resolver != nil { - d.resolver.close() + d.resolver.cancel() } } diff --git a/sandboxd/outbound/outbound.go b/sandboxd/outbound/outbound.go index 8443a79f3..697dea90d 100644 --- a/sandboxd/outbound/outbound.go +++ b/sandboxd/outbound/outbound.go @@ -248,7 +248,7 @@ func (h *Host) Reloaded() { } for _, t := range off { h.dropTun(t) - h.tunDown(t.sb.ID, h.tunState(t.sb).Sock) + h.tunDown(t.sb.ID, h.o.TunState(t.sb).Sock) } } diff --git a/sandboxd/outbound/tun.go b/sandboxd/outbound/tun.go index d34d93b74..fcd2ad2d8 100644 --- a/sandboxd/outbound/tun.go +++ b/sandboxd/outbound/tun.go @@ -91,7 +91,7 @@ func (h *Host) armTun(ctx context.Context, sb *types.Sandbox, d *doors, v *View, if h.o.Router == nil || sock == "" { return } - st := h.tunState(sb) + st := h.o.TunState(sb) if !v.Tun(key) || !st.On || !st.Addr.IsValid() { if v.Tun(key) || st.Addr.IsValid() { h.tunDown(id, sock) @@ -111,7 +111,7 @@ func (h *Host) armTun(ctx context.Context, sb *types.Sandbox, d *doors, v *View, tc := tunnet.Claim{ ID: id, Addr: t.addr, - Enabled: func() bool { return h.o.View().Tun(key) && h.tunLive(id, t) && h.tunState(sb).On }, + Enabled: func() bool { return h.o.View().Tun(key) && h.tunLive(id, t) && h.o.TunState(sb).On }, Profile: profile, Allow: func(dst netip.Addr, port uint16) bool { if c.current().EvalFlow(port) != egress.DecisionAllow { @@ -124,9 +124,7 @@ func (h *Host) armTun(ctx context.Context, sb *types.Sandbox, d *doors, v *View, Used: t.used, Touch: sb.TouchAt, Record: func(r tunnet.FlowRecord) { - if h.o.Flow != nil { - h.o.Flow(evCtx, id, tenant, profile, r) - } + h.o.Flow(evCtx, id, tenant, profile, r) if usage && r.Allowed && r.Up+r.Down > 0 { dst := netip.AddrPortFrom(r.Dst, r.Port).String() h.o.Transfer(evCtx, id, tenant, egress.Event{Host: dst, Port: r.Port, Upstream: "tun/" + cmp.Or(profile, "default")}, r.Up, r.Down) @@ -140,7 +138,7 @@ func (h *Host) armTun(ctx context.Context, sb *types.Sandbox, d *doors, v *View, d.tun = t } h.mu.Unlock() - if live && !h.tunState(sb).On { + if live && !h.o.TunState(sb).On { h.mu.Lock() mine := d.tun == t if mine { @@ -189,13 +187,6 @@ func (h *Host) serveTun(ctx context.Context, t *tunDoor) { } } -func (h *Host) tunState(sb *types.Sandbox) TunState { - if h.o.TunState == nil { - return TunState{On: !sb.TunOff, Addr: sb.TunAddr, Profile: sb.TunProfile, Bytes: sb.TunBytes, Sock: sb.VsockSocket} - } - return h.o.TunState(sb) -} - func (h *Host) tunDown(id, sock string) { go func() { ctx, cancel := context.WithTimeout(context.Background(), tunVerbTimeout) @@ -220,7 +211,5 @@ func (h *Host) dropTun(t *tunDoor) { } _ = t.ln.Close() h.o.Router.Disarm(t.addr) - if h.o.TunBytes != nil { - h.o.TunBytes(t.sb, t.used.Load()) - } + h.o.TunBytes(t.sb, t.used.Load()) } diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index e973edf77..8a490af17 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -158,11 +158,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { js, archivedAt := m.store.set(sb), time.Now() m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { - m.mu.Lock() - var rb claimSnapshot - // A Release that landed meanwhile already deleted the claim; re-setting it would resurrect it - live := m.claimed[sb.ID] == sb - if live { + rb, live := m.setIfLive(sb, func() { sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk = "", 0, prevPending, replica if replica != "" { sb.ArchiveEpoch = epoch @@ -170,9 +166,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { sb.HibernateSnap, sb.VsockSocket = snap, sock m.setVMNameLocked(sb, vmName) sb.Deadline = prevDeadline - rb = m.store.set(sb) - } - m.mu.Unlock() + }) sb.Transition.Unlock() m.store.recommit(ctx, rb) if !live || replica == "" { @@ -274,7 +268,7 @@ func (m *Manager) handOffForRoom(ctx context.Context, sb *types.Sandbox) bool { logger := log.WithFunc("pool.handOffForRoom") err := m.handOff(ctx, sb, true) switch { - case err == nil: + case err == nil || errors.Is(err, ErrUnknownSandbox): m.counters.wakeHandoffs.Add(1) logger.Infof(ctx, "handed archived %s to a peer with room to wake it", sb.ID) return true @@ -301,14 +295,10 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc js, lease := m.store.set(sb), sb.Deadline m.mu.Unlock() if err := m.store.commit(js); err != nil { - m.mu.Lock() - var rb claimSnapshot - if m.claimed[sb.ID] == sb { + rb, _ := m.setIfLive(sb, func() { sb.VsockSocket, sb.ArchiveCk, sb.ArchiveEpoch, sb.Deadline, sb.ReaderSafe = "", ck, epoch, deadline, safe m.setVMNameLocked(sb, "") - rb = m.store.set(sb) - } - m.mu.Unlock() + }) m.store.recommit(ctx, rb) return fmt.Errorf("wake %s: persist claims: %w", sb.ID, err) } diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index d964708d0..8a0a73268 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" "testing/synctest" "time" @@ -168,25 +169,8 @@ func TestReconcileReclaimsOrphanArchiveCk(t *testing.T) { } seedClaims(t, dataDir, claims) m := newTestManagerAt(t, eng, dataDir) - seedCk := func(id, sandboxID string) { - t.Helper() - staging, err := m.ckpts.Stage(id) - if err != nil { - t.Fatalf("stage: %v", err) - } - if err := os.MkdirAll(filepath.Join(staging, store.ExportDir), 0o750); err != nil { - t.Fatalf("mkdir: %v", err) - } - meta := `{"id":"` + id + `","sandbox_id":"` + sandboxID + `","archive":true}` - if err := os.WriteFile(filepath.Join(staging, store.MetaFile), []byte(meta), 0o600); err != nil { - t.Fatalf("meta: %v", err) - } - if err := m.ckpts.Publish(t.Context(), staging, id, ""); err != nil { - t.Fatalf("publish: %v", err) - } - } - seedCk("ck_00000000000000aa", "sb_mine") - seedCk("ck_00000000000000bb", "sb_elsewhere") + seedArchiveCkBy(t, m, "", "ck_00000000000000aa", "sb_mine") + seedArchiveCkBy(t, m, "", "ck_00000000000000bb", "sb_elsewhere") if err := m.Reconcile(t.Context()); err != nil { t.Fatalf("Reconcile: %v", err) @@ -388,9 +372,7 @@ func TestArchiveWakeRegrantsTheClaimsLease(t *testing.T) { if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - m.mu.Lock() - lease := time.Until(sb.Deadline) - m.mu.Unlock() + lease := time.Until(m.deadlineOf(sb)) if lease < time.Hour || lease > 2*time.Hour { t.Errorf("lease after the wake = %v, want the 2h the claim asked for", lease) } @@ -560,12 +542,8 @@ func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { } m.store.writeMu.Lock() - locked := true - t.Cleanup(func() { - if locked { - m.store.writeMu.Unlock() - } - }) + unlock := sync.OnceFunc(m.store.writeMu.Unlock) + t.Cleanup(unlock) done := make(chan error, 1) go func() { if action == testArchiveRelease { @@ -587,8 +565,7 @@ func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { t.Error("retry deleted the checkpoint before the claims commit") } breakStore(t, m) - m.store.writeMu.Unlock() - locked = false + unlock() if err := <-done; action == testArchiveRelease && err == nil { t.Fatal("release succeeded despite a persist failure") } @@ -662,12 +639,8 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { id, token, ck := sb.ID, sb.Token, sb.ArchiveCk m.store.writeMu.Lock() - locked := true - t.Cleanup(func() { - if locked { - m.store.writeMu.Unlock() - } - }) + unlock := sync.OnceFunc(m.store.writeMu.Unlock) + t.Cleanup(unlock) woke := make(chan error, 1) go func() { _, _, _, err := m.WakeAgentSocket(t.Context(), id, token) @@ -695,8 +668,7 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { t.Fatal("retry deleted the checkpoint before wake persistence completed") } breakStore(t, m) - m.store.writeMu.Unlock() - locked = false + unlock() if err := <-woke; err == nil { t.Fatal("wake succeeded despite a persist failure") } @@ -784,78 +756,71 @@ func TestArchivePersistFailureRollsBack(t *testing.T) { }) } -func TestReleaseArchivedRollsBackOnPersistFailure(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng, archivePool(3600)) - sb := mustClaim(t, m, testKey) - id, token := sb.ID, sb.Token - mustArchive(t, m, sb) - ck := sb.ArchiveCk - breakStore(t, m) - - if err := m.Release(t.Context(), id, Cred{Token: token}); err == nil { - t.Fatal("release succeeded despite a persist failure") - } - if _, ok := m.claim(id, token); !ok { - t.Error("release dropped the claim despite a failed persist") - } - if !pinnedHidden(t, m, ck) { - t.Error("kept ck is not pinned after a failed release") - } - healStore(t, m) - }) -} - -func TestWakeArchivedRollsBackOnPersistFailure(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng, archivePool(3600)) - sb := mustClaim(t, m, testKey) - id, token := sb.ID, sb.Token - mustArchive(t, m, sb) - ck := sb.ArchiveCk - breakStore(t, m) - - if _, _, _, err := m.WakeAgentSocket(t.Context(), id, token); err == nil { - t.Fatal("wake succeeded despite a persist failure") - } - m.mu.Lock() - archived := sb.ArchiveCk == ck && sb.VMName == "" - m.mu.Unlock() - if !archived { - t.Error("wake did not roll the record back to archived on a failed persist") - } - if !pinnedHidden(t, m, ck) { - t.Error("kept ck is not pinned after a failed wake") - } - healStore(t, m) - }) -} - -func TestReapPurgeRollsBackOnPersistFailure(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng, archivePool(3600)) - sb := mustClaim(t, m, testKey) - id := sb.ID - mustArchive(t, m, sb) - ck := sb.ArchiveCk - expire(m, sb) - breakStore(t, m) +func TestArchivedClaimRollsBackOnPersistFailure(t *testing.T) { + tests := []struct { + name string + setup func(m *Manager, sb *types.Sandbox) + act func(t *testing.T, m *Manager, sb *types.Sandbox, ck string) + }{ + { + name: "release", + act: func(t *testing.T, m *Manager, sb *types.Sandbox, _ string) { + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("release succeeded despite a persist failure") + } + if _, ok := m.claim(sb.ID, sb.Token); !ok { + t.Error("release dropped the claim despite a failed persist") + } + }, + }, + { + name: "wake", + act: func(t *testing.T, m *Manager, sb *types.Sandbox, ck string) { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { + t.Fatal("wake succeeded despite a persist failure") + } + m.mu.Lock() + archived := sb.ArchiveCk == ck && sb.VMName == "" + m.mu.Unlock() + if !archived { + t.Error("wake did not roll the record back to archived on a failed persist") + } + }, + }, + { + name: "reap purge", + setup: expire, + act: func(t *testing.T, m *Manager, sb *types.Sandbox, _ string) { + m.reapOnce(t.Context()) + m.mu.Lock() + _, present := m.claimed[sb.ID] + m.mu.Unlock() + if !present { + t.Error("reap dropped the archived claim despite a failed persist") + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + m := newTestManager(t, newFakeEngine(), archivePool(3600)) + sb := mustClaim(t, m, testKey) + mustArchive(t, m, sb) + ck := sb.ArchiveCk + if tt.setup != nil { + tt.setup(m, sb) + } + breakStore(t, m) - m.reapOnce(t.Context()) - m.mu.Lock() - _, present := m.claimed[id] - m.mu.Unlock() - if !present { - t.Error("reap dropped the archived claim despite a failed persist") - } - if !pinnedHidden(t, m, ck) { - t.Error("kept ck is not pinned after a failed reap purge") - } - healStore(t, m) - }) + tt.act(t, m, sb, ck) + if !pinnedHidden(t, m, ck) { + t.Errorf("kept ck is not pinned after a failed %s", tt.name) + } + healStore(t, m) + }) + }) + } } func TestArchiveOnceSkipsInFlight(t *testing.T) { diff --git a/sandboxd/pool/capture.go b/sandboxd/pool/capture.go index ad3c4a096..964016e10 100644 --- a/sandboxd/pool/capture.go +++ b/sandboxd/pool/capture.go @@ -8,6 +8,7 @@ import ( "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/outbound" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -57,37 +58,37 @@ func (m *Manager) captureAllowed(sb *types.Sandbox) error { return nil } -func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir, from string) (snap, vmName string, guestEnv bool, err error) { +func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir, from string) (snap, vmName string, guestEnv bool, tun outbound.TunState, err error) { if lockErr := m.lockTransition(sb); lockErr != nil { - return "", "", false, lockErr + return "", "", false, outbound.TunState{}, lockErr } if from != "" && sb.HibernateSnap != from { sb.Transition.Unlock() - return "", "", false, errWokeMeanwhile + return "", "", false, outbound.TunState{}, errWokeMeanwhile } if captureErr := m.captureAllowed(sb); captureErr != nil { sb.Transition.Unlock() - return "", "", false, captureErr + return "", "", false, outbound.TunState{}, captureErr } if settleErr := m.settleCapture(ctx, sb); settleErr != nil { sb.Transition.Unlock() - return "", "", false, settleErr + return "", "", false, outbound.TunState{}, settleErr } m.trimForCapture(ctx, sb) snap, cleanup, err := m.sourceSnap(ctx, sb) if err != nil { sb.Transition.Unlock() - return "", "", false, err + return "", "", false, outbound.TunState{}, err } defer cleanup() - vmName, guestEnv = sb.VMName, m.heldGuestEnv(sb) + vmName, guestEnv, tun = sb.VMName, m.heldGuestEnv(sb), m.tunState(sb) // A fresh snapshot is this capture's alone, so dials need not wait out its export; a wake consumes a hibernate snapshot if snap != sb.HibernateSnap { sb.Transition.Unlock() } else { defer sb.Transition.Unlock() } - return snap, vmName, guestEnv, m.eng.SnapshotExport(ctx, snap, exportDir) + return snap, vmName, guestEnv, tun, m.eng.SnapshotExport(ctx, snap, exportDir) } // A failed trim only leaves the capture larger. diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 62c44950c..8065fd89c 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -171,7 +171,6 @@ func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, [ } func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID, name, tenant, from string, archive bool) (types.Checkpoint, string, error) { - tun := m.tunState(sb) ckpt := types.Checkpoint{ ID: ckID, Name: name, @@ -181,8 +180,6 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID CreatedAt: time.Now(), PolicySource: sb.PolicySource, NoEgress: sb.NoEgress, - TunOff: !tun.On, - TunProfile: tun.Profile, Archive: archive, } if archive { @@ -193,11 +190,11 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID return types.Checkpoint{}, "", fmt.Errorf("stage checkpoint: %w", err) } defer func() { _ = os.RemoveAll(staging) }() - srcSnap, _, guestEnv, err := m.exportSource(ctx, sb, filepath.Join(staging, store.ExportDir), from) + srcSnap, _, guestEnv, tun, err := m.exportSource(ctx, sb, filepath.Join(staging, store.ExportDir), from) if err != nil { return types.Checkpoint{}, "", fmt.Errorf("checkpoint %s: %w", sb.ID, err) } - ckpt.GuestEnv = guestEnv + ckpt.GuestEnv, ckpt.TunOff, ckpt.TunProfile = guestEnv, !tun.On, tun.Profile meta, err := json.Marshal(ckpt) if err != nil { return types.Checkpoint{}, "", fmt.Errorf("encode checkpoint meta: %w", err) @@ -217,12 +214,12 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, o Clai return nil, err } defer m.recs.rlockRec(ckpt.ID)() - if ckpt.Archive { + if ckpt.Archive || !tenantOwns(o.Tenant, ckpt.Tenant) { return nil, ErrUnknownCheckpoint // a wake image, not a branchable checkpoint } dir, _, _, err := m.ckpts.Fetch(ctx, ckpt.ID) if errors.Is(err, store.ErrNotFound) { - return nil, ErrUnknownCheckpoint // deleted between the pre-check and the lock + return nil, ErrUnknownCheckpoint } if err != nil { return nil, fmt.Errorf("fetch checkpoint: %w", err) @@ -368,9 +365,8 @@ func (h *healVetoes) clear(ckptID string) (aborted bool) { h.mu.Lock() defer h.mu.Unlock() delete(h.pending, ckptID) - if _, aborted = h.aborted[ckptID]; aborted { - delete(h.aborted, ckptID) - } + _, aborted = h.aborted[ckptID] + delete(h.aborted, ckptID) return aborted } diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index 7611abe75..1399fc9ef 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -329,61 +329,36 @@ func TestValidateHealedCheckpoint(t *testing.T) { } } -func TestDeleteCheckpointBroadcasts(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng) - src := mustClaim(t, m, testKey) - ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") - if err != nil { - t.Fatalf("checkpoint: %v", err) - } - var calls []string - m.peerDelete = func(_ context.Context, id string) { calls = append(calls, id) } - - if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "", DeleteFleet); err != nil { - t.Fatalf("DeleteCheckpoint: %v", err) - } - if len(calls) != 1 || calls[0] != ckpt.ID { - t.Errorf("peerDelete calls = %v, want one call for %s", calls, ckpt.ID) - } -} - -func TestDeleteCheckpointNoForwardSkipsBroadcast(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng) - src := mustClaim(t, m, testKey) - ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") - if err != nil { - t.Fatalf("checkpoint: %v", err) - } - called := false - m.peerDelete = func(context.Context, string) { called = true } - - if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "", DeleteLocal); err != nil { - t.Fatalf("DeleteCheckpoint: %v", err) - } - if called { - t.Error("peerDelete called despite no_forward") - } -} - -func TestDeleteCheckpointSharedStoreSkipsBroadcast(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng) - src := mustClaim(t, m, testKey) - ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") - if err != nil { - t.Fatalf("checkpoint: %v", err) +func TestDeleteCheckpointBroadcastsOnlyAcrossAFleet(t *testing.T) { + tests := []struct { + name string + scope DeleteScope + shared bool + calls int + }{ + {"fleet scope", DeleteFleet, false, 1}, + {"no forward", DeleteLocal, false, 0}, + {"shared store", DeleteFleet, true, 0}, } - m.ckptsShared = true - called := false - m.peerDelete = func(context.Context, string) { called = true } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + src := mustClaim(t, m, testKey) + ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + m.ckptsShared = tt.shared + var calls []string + m.peerDelete = func(_ context.Context, id string) { calls = append(calls, id) } - if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "", DeleteFleet); err != nil { - t.Fatalf("DeleteCheckpoint: %v", err) - } - if called { - t.Error("peerDelete called on a shared-store node") + if err = m.DeleteCheckpoint(t.Context(), ckpt.ID, "", tt.scope); err != nil { + t.Fatalf("DeleteCheckpoint: %v", err) + } + if len(calls) != tt.calls || tt.calls == 1 && calls[0] != ckpt.ID { + t.Errorf("peerDelete calls = %v, want %d for %s", calls, tt.calls, ckpt.ID) + } + }) } } diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index 061cf344a..e85887925 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -277,6 +277,12 @@ func TestCheckpointTenantIsolation(t *testing.T) { t.Errorf("root listing %+v, %v; want both records", ckpts, err) } + if _, err := m.ClaimCheckpoint(t.Context(), ckA.ID, ClaimOptions{Tenant: "beta"}); !errors.Is(err, ErrUnknownCheckpoint) { + t.Errorf("cross-tenant branch: %v, want ErrUnknownCheckpoint", err) + } + if _, err := m.ClaimCheckpoint(t.Context(), ckA.ID, ClaimOptions{Tenant: "acme"}); err != nil { + t.Errorf("own branch: %v", err) + } if err := m.DeleteCheckpoint(t.Context(), ckA.ID, "beta", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("cross-tenant delete: %v, want ErrUnknownCheckpoint", err) } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index d5301b28f..5eeeef992 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -12,6 +12,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" "github.com/cocoonstack/sandbox/sandboxd/usage" + "github.com/cocoonstack/sandbox/sandboxd/utils" ) // ClaimOptions is what a claim asks for beyond its key; the zero value takes every default. @@ -626,7 +627,7 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t m.rollbackClaim(ctx, sbs, false) return fmt.Errorf("persist claim: %w", saveErr) } - // The claim event follows the commit at once, so a crash cannot restore a claim nothing billed + // The claim event follows the commit, so a claim that never committed is never billed for _, sb := range sbs { m.recordUsage(ctx, usage.Event{ Event: "claim", @@ -663,10 +664,8 @@ func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox, bille m.recordUsage(ctx, usage.Event{Event: "release", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant}) } } - for _, sb := range sbs { - m.releaseTun(ctx, sb.ID, sb.TunAddr) - } for i, sb := range sbs { + m.releaseTun(ctx, sb.ID, sb.TunAddr) td := m.quiesceVolumes(ctx, sb) td.holds = holds[i] m.out.Disarm(sb.ID, m.removeOrRetry(ctx, sb.VMName, sb.ID, td)) @@ -737,11 +736,8 @@ func (c *heldConn) Close() error { // CloseWrite forwards the half-close; net.Conn does not carry it, so embedding cannot promote one. func (c *heldConn) CloseWrite() error { - cw, ok := c.Conn.(interface{ CloseWrite() error }) - if !ok { - return nil - } - return cw.CloseWrite() + utils.CloseWrite(c.Conn) + return nil } func clampTTL(ttl time.Duration) time.Duration { diff --git a/sandboxd/pool/cordon_test.go b/sandboxd/pool/cordon_test.go index 3a33bd546..40cf55e4f 100644 --- a/sandboxd/pool/cordon_test.go +++ b/sandboxd/pool/cordon_test.go @@ -175,9 +175,7 @@ func TestACordonedNodeNeverAdoptsButKeepsItsOwnRowAcrossARestart(t *testing.T) { mustArchive(t, a, lost) own := mustClaim(t, b, testKey) mustArchive(t, b, own) - if err := b.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, b) if err := b.Cordon(t.Context()); err != nil { t.Fatalf("cordon: %v", err) } @@ -227,10 +225,7 @@ func TestCordonOutlivesAReloadAndAnUndrain(t *testing.T) { if err := m.Cordon(t.Context()); err != nil { t.Fatalf("cordon: %v", err) } - next := *m.cfg - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(*config.Config) {}) m.Drain(t.Context()) m.Undrain(t.Context()) if _, g := m.Info(); !g.Cordoned || g.Draining { diff --git a/sandboxd/pool/disk_test.go b/sandboxd/pool/disk_test.go index c2d462210..166c20cc8 100644 --- a/sandboxd/pool/disk_test.go +++ b/sandboxd/pool/disk_test.go @@ -2,6 +2,7 @@ package pool import ( "errors" + "path/filepath" "testing" "time" @@ -25,9 +26,7 @@ func TestDiskUnderReserveRefusesClaimsAndWakesUntilItRecovers(t *testing.T) { t.Errorf("gauges %+v, want at capacity for the disk", g) } - next := &config.Config{DataDir: m.dataDir, DiskReserve: &config.DiskReserve{Free: "1K"}, Pools: []config.PoolSpec{{PoolKey: testKey}}} - m.view.Store(newConfigView(next, testSecrets(t))) - m.checkDisk(t.Context()) + setDiskReserve(t, m, &config.DiskReserve{Free: "1K"}) if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Errorf("wake above the reserve: %v", err) } @@ -68,8 +67,7 @@ func TestDiskUnderReserveHoldsHibernatesAndCaptures(t *testing.T) { t.Errorf("promote under the reserve: %v, want ErrQuota", err) } - m.view.Store(newConfigView(&config.Config{DataDir: m.dataDir, DiskReserve: &config.DiskReserve{Free: "1K"}, Pools: []config.PoolSpec{{PoolKey: testKey}}}, testSecrets(t))) - m.checkDisk(t.Context()) + setDiskReserve(t, m, &config.DiskReserve{Free: "1K"}) if err := m.Hibernate(t.Context(), sb.ID, cred); err != nil { t.Errorf("hibernate above the reserve: %v", err) } @@ -78,14 +76,19 @@ func TestDiskUnderReserveHoldsHibernatesAndCaptures(t *testing.T) { func TestAFailedStatfsKeepsTheDiskLowState(t *testing.T) { m := diskReserveManager(t, "1000P") m.checkDisk(t.Context()) - unreadable := &config.Config{DataDir: m.dataDir, DiskReserve: &config.DiskReserve{Free: "1K", Dirs: []string{m.dataDir + "/missing"}}, Pools: []config.PoolSpec{{PoolKey: testKey}}} - m.view.Store(newConfigView(unreadable, testSecrets(t))) - m.checkDisk(t.Context()) + setDiskReserve(t, m, &config.DiskReserve{Free: "1K", Dirs: []string{filepath.Join(m.dataDir, "missing")}}) if _, err := claimAny(t.Context(), m, testKey, ClaimOptions{TTL: time.Hour}); !errors.Is(err, ErrQuota) { t.Errorf("claim after a failed statfs: %v, want ErrQuota while the disk was last seen low", err) } } +func setDiskReserve(t *testing.T, m *Manager, reserve *config.DiskReserve) { + t.Helper() + cfg := &config.Config{DataDir: m.dataDir, DiskReserve: reserve, Pools: []config.PoolSpec{{PoolKey: testKey}}} + m.view.Store(newConfigView(cfg, testSecrets(t))) + m.checkDisk(t.Context()) +} + func diskReserveManager(t *testing.T, free string) *Manager { t.Helper() cfg := &config.Config{DataDir: t.TempDir(), DiskReserve: &config.DiskReserve{Free: free}, Pools: []config.PoolSpec{{PoolKey: testKey}}} diff --git a/sandboxd/pool/drain_test.go b/sandboxd/pool/drain_test.go index 376ab667e..251309fa0 100644 --- a/sandboxd/pool/drain_test.go +++ b/sandboxd/pool/drain_test.go @@ -62,7 +62,7 @@ func TestDrainRefusesClaimsTrimsWarmAndUndrainRefills(t *testing.T) { func TestEvacuateHandsClaimsToAPeerWhileItsLeaseIsLive(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) running, asleep, archived := mustClaim(t, a, testKey), mustClaim(t, a, testKey), mustClaim(t, a, testKey) hibernateClaim(t, a, asleep) mustArchive(t, a, archived) @@ -95,7 +95,7 @@ func TestEvacuateHandsClaimsToAPeerWhileItsLeaseIsLive(t *testing.T) { func TestEvacuateHandOffRefusedByAConcurrentOwnerWriteKeepsTheClaim(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, beforeRelease: func() { @@ -115,7 +115,7 @@ func TestEvacuateHandOffRefusedByAConcurrentOwnerWriteKeepsTheClaim(t *testing.T if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Owner != "a" { t.Errorf("row owner %q after a refused hand-off, want a", rec.Owner) } - assertNotAdopted(t, b, sb, "adopt a row the hand-off did not release") + mustNotAdopt(t, b, sb, "adopt a row the hand-off did not release") a.Evacuate(t.Context()) synctest.Wait() @@ -127,7 +127,7 @@ func TestEvacuateHandOffRefusedByAConcurrentOwnerWriteKeepsTheClaim(t *testing.T func TestEvacuateHandOffOfARowAPeerTookDropsTheClaim(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, beforeRelease: func() { @@ -189,8 +189,8 @@ func TestEvacuateWithoutArchiveFailoverSkipsEveryClaim(t *testing.T) { synctest.Wait() assertEvacuation(t, m, 0, 1) - if held, ok := m.byID(sb.ID); !ok || held.HibernateSnap != "" || len(eng.hibernates) != 0 { - t.Errorf("held %v hibernates %v, want the claim left running", ok, eng.hibernates) + if !runningKept(m, sb.ID) || len(eng.hibernates) != 0 { + t.Errorf("hibernates %v, want the claim left running", eng.hibernates) } }) } @@ -198,7 +198,7 @@ func TestEvacuateWithoutArchiveFailoverSkipsEveryClaim(t *testing.T) { func TestEvacuateWaitsOutAnArchiveInFlight(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.mu.Lock() @@ -220,7 +220,7 @@ func TestEvacuateWaitsOutAnArchiveInFlight(t *testing.T) { func TestUndrainStopsAnEvacuationBeforeItsHandOff(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) stall := stallExports(a) sb := mustClaim(t, a, testKey) @@ -232,7 +232,7 @@ func TestUndrainStopsAnEvacuationBeforeItsHandOff(t *testing.T) { close(stall) synctest.Wait() - assertEvacuation(t, a, 0, 0) + assertEvacuation(t, a, 0, 1) if _, g := a.Info(); g.Draining { t.Error("still draining after undrain") } @@ -243,7 +243,7 @@ func TestUndrainStopsAnEvacuationBeforeItsHandOff(t *testing.T) { if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Owner != "a" { t.Errorf("row owner %q after undrain, want a", rec.Owner) } - assertNotAdopted(t, b, sb, "adopt a claim whose evacuation stopped") + mustNotAdopt(t, b, sb, "adopt a claim whose evacuation stopped") if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Errorf("wake after undrain: %v", err) } @@ -273,10 +273,9 @@ func TestAnUndrainDuringAnEvacuationsDrainStopsIt(t *testing.T) { wg.Wait() synctest.Wait() - assertEvacuation(t, a, 0, 0) - held, ok := a.byID(sb.ID) - if !ok || held.HibernateSnap != "" || held.ArchiveCk != "" { - t.Errorf("held %v, want the claim still running after the undrain", ok) + assertEvacuation(t, a, 0, 1) + if !runningKept(a, sb.ID) { + t.Error("want the claim still running after the undrain") } }) } @@ -381,8 +380,8 @@ func TestEvacuateKeepsAClaimOfARemovedTenant(t *testing.T) { synctest.Wait() assertEvacuation(t, m, 0, 1) - if held, ok := m.byID(sb.ID); !ok || held.HibernateSnap != "" || held.ArchiveCk != "" { - t.Errorf("held %v, want the claim left running", ok) + if !runningKept(m, sb.ID) { + t.Error("want the claim left running") } }) } @@ -405,7 +404,7 @@ func TestAnEnvWriteThatLostTheClaimAnswersUnknown(t *testing.T) { func TestAnEnvWriteDuringTheArchiveReachesItsRow(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) written := make(chan error, 1) @@ -452,8 +451,8 @@ func TestASecondEvacuateWhileOneRunsChangesNothing(t *testing.T) { a.Undrain(t.Context()) release() synctest.Wait() - if held, ok := a.byID(sb.ID); !ok || held.HibernateSnap != "" || held.ArchiveCk != "" { - t.Errorf("held %v, want the claim left running after undrain", ok) + if !runningKept(a, sb.ID) { + t.Error("want the claim left running after undrain") } }) } @@ -471,8 +470,8 @@ func TestAStoppedEvacuationDoesNotPauseAClaimItHadQueued(t *testing.T) { if err := a.evacuateClaim(t.Context(), sb); err != nil { t.Errorf("evacuateClaim after undrain: %v, want nil", err) } - if held, ok := a.byID(sb.ID); !ok || held.HibernateSnap != "" || held.ArchiveCk != "" { - t.Errorf("held %v, want the claim left running", ok) + if !runningKept(a, sb.ID) { + t.Error("want the claim left running") } release() @@ -507,7 +506,7 @@ func TestEvacuateKeepsAClaimArchivedWithoutARow(t *testing.T) { func TestEvacuateMovesTheCheapestClaimsFirst(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) running, hibernated, archived := mustClaim(t, a, testKey), mustClaim(t, a, testKey), mustClaim(t, a, testKey) hibernateClaim(t, a, hibernated) mustArchive(t, a, archived) @@ -580,7 +579,7 @@ func TestEvacuateWithNoPeerToAdoptKeepsTheClaim(t *testing.T) { func TestEvacuateConvergesWhenAPeersAnswerIsLost(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) a.WithPeerAdopt(func(ctx context.Context, id, token string, _ bool) bool { if _, err := b.AdoptArchived(ctx, id, token); err != nil { @@ -605,7 +604,7 @@ func TestEvacuateConvergesWhenAPeersAnswerIsLost(t *testing.T) { func TestEvacuateConvergesWhenTheReleaseReplyIsLost(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) a.archives = &hookRegistry{Registry: c.reg, loseReleaseReply: true} @@ -626,7 +625,7 @@ func TestEvacuateConvergesWhenTheReleaseReplyIsLost(t *testing.T) { func TestARepeatEvacuateResumesAHandOffAnOutageCutShort(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) a.archives = &hookRegistry{Registry: c.reg, loseReleaseReply: true} @@ -649,7 +648,7 @@ func TestARepeatEvacuateResumesAHandOffAnOutageCutShort(t *testing.T) { func TestARepeatEvacuateWithNoAdopterLeavesAReleasedRowReleased(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) a.archives = &hookRegistry{Registry: c.reg, loseReleaseReply: true} a.Evacuate(t.Context()) @@ -673,7 +672,7 @@ func TestARepeatEvacuateWithNoAdopterLeavesAReleasedRowReleased(t *testing.T) { func TestACrashBeforeThePeersAnswerLeavesARowAnyNodeSettles(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) adopted, purged := mustClaim(t, a, testKey), mustClaim(t, a, testKey) for _, sb := range []*types.Sandbox{adopted, purged} { mustArchive(t, a, sb) @@ -747,11 +746,16 @@ func TestEvacuationDeadlines(t *testing.T) { func assertEvacuation(t *testing.T, m *Manager, left, kept int) { t.Helper() - if _, g := m.Info(); g.Evacuating != left || g.Draining && g.Claimed-g.Evacuating != kept { + if _, g := m.Info(); g.Evacuating != left || g.Claimed-g.Evacuating != kept { t.Errorf("evacuation left %d of %d claims, want %d left and %d kept", g.Evacuating, g.Claimed, left, kept) } } +func runningKept(m *Manager, id string) bool { + held, ok := m.byID(id) + return ok && held.HibernateSnap == "" && held.ArchiveCk == "" +} + func holdMaint(m *Manager) (release func()) { for range cap(m.maintSem) { m.maintSem <- struct{}{} diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index f94f56641..79fb6605e 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -112,7 +112,7 @@ func TestEffectivePolicyKeepsTheLayerPinnedAtClaim(t *testing.T) { cfg.EgressClasses = []config.EgressClass{{Name: "desk", Egress: &egress.Policy{Allow: []egress.Rule{{Host: "b.test"}}}}} }) - m.pools = map[types.PoolKey]*pool{testKey: newPool(testKey)} + m.pools = map[types.PoolKey]*pool{testKey: newPool(config.PoolSpec{PoolKey: testKey})} if _, ok := m.view.Load().out.Resolve(&types.Sandbox{Key: testKey, Tenant: "acme", EgressClass: "desk", Layer: types.LayerUnpooled}); !ok { t.Error("a claim made on an unpooled key lost its tenant policy when the key gained a pool") } @@ -252,9 +252,7 @@ func TestBatchArmFailureEndsEveryClaimEvent(t *testing.T) { t.Fatal("finalizeBatch must fail when a batch member cannot arm") } waitFor(t, m.store.synced) - m.mu.Lock() - claimed := len(m.claimed) - m.mu.Unlock() + claimed := claimedCount(m) if claimed != 0 { t.Errorf("rollback left %d claims", claimed) } @@ -293,9 +291,7 @@ func TestRollbackKeepsLockWhenRemoveFails(t *testing.T) { m.rollbackClaim(t.Context(), sbs, false) waitFor(t, m.store.synced) - m.mu.Lock() - claimed := len(m.claimed) - m.mu.Unlock() + claimed := claimedCount(m) if claimed != 0 { t.Errorf("rollback left %d claims", claimed) } @@ -534,7 +530,7 @@ func TestAPromotedTemplatesCloneInterceptsAsItsSourcePool(t *testing.T) { func TestAClaimWithoutEgressGetsNoDoorsWhateverItsPolicy(t *testing.T) { eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) - pol := &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + pol := hostPolicy() m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: pol}) pin := pinDoor(t, engine.EgressSocketPath(refillWarmVM(t, m).VsockSocket)) warm, err := m.ClaimWarm(t.Context(), testKey, ClaimOptions{NoEgress: true}) diff --git a/sandboxd/pool/env_test.go b/sandboxd/pool/env_test.go index d5b0720e4..4ed0ad221 100644 --- a/sandboxd/pool/env_test.go +++ b/sandboxd/pool/env_test.go @@ -349,9 +349,7 @@ func TestAFailedEnvDeliveryDestroysTheClaim(t *testing.T) { if len(eng.removes) != 1 { t.Errorf("removed %v, want the claim's VM", eng.removes) } - m.mu.Lock() - claimed := len(m.claimed) - m.mu.Unlock() + claimed := claimedCount(m) if claimed != 0 { t.Errorf("%d claims recorded, want none", claimed) } @@ -502,9 +500,7 @@ func TestAClaimWithAnUninterceptedInjectIsRefused(t *testing.T) { if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Env: injectEnv("k", "other.test")}); !errors.Is(err, ErrBadEnv) { t.Fatalf("claim: %v, want ErrBadEnv", err) } - m.mu.Lock() - claimed := len(m.claimed) - m.mu.Unlock() + claimed := claimedCount(m) if claimed != 0 || len(eng.removedNames()) != 1 { t.Errorf("%d claims left and %v removed, want none kept and the VM destroyed", claimed, eng.removedNames()) } diff --git a/sandboxd/pool/expiry_test.go b/sandboxd/pool/expiry_test.go index 92e652a8c..1e9768781 100644 --- a/sandboxd/pool/expiry_test.go +++ b/sandboxd/pool/expiry_test.go @@ -32,9 +32,7 @@ func TestExpiryArchiveHibernatesAndArchivesARunningClaim(t *testing.T) { if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("round %d: wake: %v", round, err) } - m.mu.Lock() - lease := time.Until(sb.Deadline) - m.mu.Unlock() + lease := time.Until(m.deadlineOf(sb)) if lease < 59*time.Minute { t.Fatalf("round %d: lease after the wake %v, want the claim's hour again", round, lease) } @@ -174,9 +172,7 @@ func TestExpiryArchiveWakeBeforeTheExportGrantsAFreshLease(t *testing.T) { if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - m.mu.Lock() - lease := time.Until(sb.Deadline) - m.mu.Unlock() + lease := time.Until(m.deadlineOf(sb)) if lease < 59*time.Minute { t.Fatalf("lease after a wake from the expiry hibernate = %v, want the claim's hour again", lease) } @@ -200,9 +196,7 @@ func TestATierClaimWokenPastItsLeaseBeforeTheExportGetsAFreshLease(t *testing.T) if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - m.mu.Lock() - lease := time.Until(sb.Deadline) - m.mu.Unlock() + lease := time.Until(m.deadlineOf(sb)) if lease < 59*time.Minute { t.Fatalf("lease after a wake past the lease = %v, want the claim's hour again", lease) } diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 6336d6243..efbe621a5 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -118,11 +118,6 @@ func (m *Manager) replaceArchive(ctx context.Context, sb *types.Sandbox, ck stri return m.putRow(ctx, sb, ck, false, true) } -// putReplica writes a live claim's row with no deadline: retention starts when the claim is archived or adopted. -func (m *Manager) putReplica(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { - return m.putRow(ctx, sb, ck, true, false) -} - // replaceReplica moves the row from the epoch this node holds; a row another node took or deleted refuses the write. func (m *Manager) replaceReplica(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { return m.putRow(ctx, sb, ck, true, true) @@ -324,13 +319,7 @@ func (m *Manager) dropTombstone(ctx context.Context, id string, epoch int64) { } func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox) { - var epoch int64 - var err error - if sb.ArchiveCk != "" { - epoch, err = m.putArchive(ctx, sb, sb.ArchiveCk) - } else { - epoch, err = m.putReplica(ctx, sb, sb.ReplicaCk) - } + epoch, err := m.putRow(ctx, sb, cmp.Or(sb.ArchiveCk, sb.ReplicaCk), sb.ArchiveCk == "", false) if err != nil { if !errors.Is(err, ErrVolumeCapture) { log.WithFunc("pool.reputArchive").Warnf(ctx, "re-put archive row %s: %v", sb.ID, err) diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 42066c7f0..106a50932 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -72,7 +72,7 @@ func TestFailoverRowWriteFailureKeepsTheClaimHibernated(t *testing.T) { func TestFailoverEnvRotationReachesTheAdopter(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := archivedWithEnv(t, a) if rowEnv(t, c, sb) != "old" { t.Fatalf("archive row env %q, want old", rowEnv(t, c, sb)) @@ -150,7 +150,7 @@ func TestFailoverEnvRotationSurvivesALostReply(t *testing.T) { func TestFailoverEnvRotationOfAClaimAdoptedElsewhereDropsIt(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := archivedWithEnv(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") @@ -243,7 +243,7 @@ func TestFailoverOwnerWakeWithTheRegistryDownFailsClosed(t *testing.T) { func TestFailoverAdoptionMovesTheClaimAndTheOldOwnerLetsGo(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -273,11 +273,11 @@ func TestFailoverAdoptionMovesTheClaimAndTheOldOwnerLetsGo(t *testing.T) { func TestFailoverNoAdoptionWhileTheOwnerIsAlive(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - assertNotAdopted(t, b, sb, "adopt from a live owner") + mustNotAdopt(t, b, sb, "adopt from a live owner") if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Owner != "a" || rec.Epoch != 1 { t.Errorf("row %s@%d, want a@1", rec.Owner, rec.Epoch) } @@ -285,13 +285,13 @@ func TestFailoverNoAdoptionWhileTheOwnerIsAlive(t *testing.T) { func TestFailoverADrainingNodeDoesNotAdopt(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.expire("a") b.Drain(t.Context()) - assertNotAdopted(t, b, sb, "adopt on a draining node") + mustNotAdopt(t, b, sb, "adopt on a draining node") if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Owner != "a" || rec.Epoch != 1 { t.Errorf("row %s@%d, want a@1 left for another node", rec.Owner, rec.Epoch) } @@ -303,7 +303,7 @@ func TestFailoverADrainingNodeDoesNotAdopt(t *testing.T) { func TestFailoverAdoptionIgnoresClaimCapsAndAWakeStillChecksMaxRunning(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) for _, m := range []*Manager{a, b} { if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "acme", Token: "tok-acme", MaxClaims: 1}); err != nil { t.Fatalf("put tenant: %v", err) @@ -339,7 +339,7 @@ func TestFailoverAdoptionIgnoresClaimCapsAndAWakeStillChecksMaxRunning(t *testin func TestFailoverAdoptionNeedsTheToken(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.expire("a") @@ -493,9 +493,7 @@ func TestFailoverRestartDropsAClaimWhoseInterruptedArchiveWasAdopted(t *testing. a := c.node(t, "a", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) a.mu.Lock() vm, snap := sb.VMName, sb.HibernateSnap a.mu.Unlock() @@ -527,9 +525,7 @@ func TestFailoverRestartWithTheRegistryDownKeepsEverything(t *testing.T) { a := c.node(t, "a", dirA, keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.reg.SetDown(true) restarted := c.node(t, "a", dirA, keepAnHour) @@ -566,7 +562,7 @@ func TestFailoverReleaseDropsTheRowThenTheCheckpoint(t *testing.T) { func TestFailoverReleaseAfterAdoptionKeepsTheCheckpoint(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.expire("a") @@ -640,7 +636,7 @@ func TestFailoverSweepPurgesADepartedOwnersArchiveOnce(t *testing.T) { func TestFailoverAdoptionMidWakeFencesTheOwnerOut(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -730,9 +726,7 @@ func TestFailoverRestartFencesWithOneQuery(t *testing.T) { mustArchive(t, a, sb) ids = append(ids, sb.ID) } - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) restarted := c.node(t, "a", dirA, keepAnHour) counted := &countingRegistry{Registry: c.reg} restarted.archives = counted @@ -799,9 +793,7 @@ func TestFailoverRestartAfterAConsumeWithoutCommitKeepsTheClaim(t *testing.T) { epoch := c.take(t, sb.ID, "a") _, _ = a.commitIfLive(t.Context(), sb, func() { sb.ArchiveEpoch = epoch }) c.consume(t, sb.ID, epoch) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) restarted := c.restartReconciled(t, a) held, _ := restarted.byID(sb.ID) @@ -819,9 +811,7 @@ func TestFailoverAnArchiveWokenElsewhereIsDropped(t *testing.T) { a, b := c.node(t, "a", dirA, keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { @@ -843,7 +833,7 @@ func TestFailoverAnArchiveWokenElsewhereIsDropped(t *testing.T) { func TestFailoverALeftoverDeleteMarkerSparesTheAdoptersCheckpoint(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -864,16 +854,14 @@ func TestFailoverALeftoverDeleteMarkerSparesTheAdoptersCheckpoint(t *testing.T) func TestFailoverRestartClearsTheMarkerOfAnAdoptedClaim(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk if err := a.ckMarks.mark(ck); err != nil { t.Fatalf("mark: %v", err) } - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") c.renew(t, "a") @@ -886,7 +874,7 @@ func TestFailoverRestartClearsTheMarkerOfAnAdoptedClaim(t *testing.T) { func TestFailoverARepeatedAdoptionOnOneNodeKeepsOneClaim(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) for _, m := range []*Manager{a, b} { if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "acme", Token: "tok-acme"}); err != nil { t.Fatalf("put tenant: %v", err) @@ -927,12 +915,10 @@ func TestFailoverARepeatedAdoptionOnOneNodeKeepsOneClaim(t *testing.T) { func TestFailoverRestartInsideAnAdoptersWakeDropsTheClaim(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) ck := sb.ArchiveCk c.expire("a") mustAdopt(t, b, sb, "adopt") @@ -961,12 +947,10 @@ func TestFailoverRestartInsideAnAdoptersWakeDropsTheClaim(t *testing.T) { func TestFailoverRestartAfterAnAdopterReleasedDropsTheClaim(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") b.ckpts = &failingDeleteStore{Store: b.ckpts, attempts: make(chan string, 8)} @@ -1020,12 +1004,10 @@ func TestFailoverReleaseDuringAnArchiveLeavesNoRow(t *testing.T) { func TestFailoverRestartDropsAClaimAdoptedWhileItsInterruptedArchiveSettles(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) const ck = "ck_00000000000000cc" seedArchiveCk(t, a, ck, sb.ID) if _, err := a.putArchive(t.Context(), sb, ck); err != nil { @@ -1055,9 +1037,7 @@ func TestFailoverRestartKeepsAClaimWhoseInterruptedArchiveRowVanished(t *testing a := c.node(t, "a", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) const ck = "ck_00000000000000dd" seedArchiveCk(t, a, ck, sb.ID) epoch, err := a.putArchive(t.Context(), sb, ck) @@ -1091,9 +1071,7 @@ func TestFailoverRestartLeavesItsOwnTombstoneToTheSweep(t *testing.T) { if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) before, _, _ := c.reg.Get(t.Context(), sb.ID) restarted := c.restartReconciled(t, a) @@ -1107,7 +1085,7 @@ func TestFailoverRestartLeavesItsOwnTombstoneToTheSweep(t *testing.T) { func TestFailoverSweepLeavesADepartedOwnersTombstoneUntilItReturns(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) d := c.node(t, "d", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) @@ -1205,7 +1183,7 @@ func TestFailoverATombstoneIsNotAdopted(t *testing.T) { counted := &countingRegistry{Registry: c.reg} d.archives = counted - assertNotAdopted(t, d, sb, "adopt of a tombstone") + mustNotAdopt(t, d, sb, "adopt of a tombstone") if n := counted.takes.Load(); n != 0 { t.Errorf("adoption tried %d takes on a tombstone, want none", n) } @@ -1214,7 +1192,7 @@ func TestFailoverATombstoneIsNotAdopted(t *testing.T) { func TestFailoverADropClearsTheMarkerBeforeTheCheckpointIsUnpinned(t *testing.T) { for range 20 { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -1249,7 +1227,7 @@ func TestFailoverADropClearsTheMarkerBeforeTheCheckpointIsUnpinned(t *testing.T) func TestFailoverReleaseAfterALostTakeReplyLeavesNoLiveRow(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, loseTakeReply: true} @@ -1264,7 +1242,7 @@ func TestFailoverReleaseAfterALostTakeReplyLeavesNoLiveRow(t *testing.T) { func TestFailoverReleaseAfterALostRearmReplyLeavesNoLiveRow(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, loseConsumeReply: true} @@ -1286,7 +1264,7 @@ func TestFailoverReleaseAfterALostRearmReplyLeavesNoLiveRow(t *testing.T) { func TestFailoverARepeatedAdoptionOfAReleasedClaimIsRefused(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.expire("a") @@ -1297,7 +1275,7 @@ func TestFailoverARepeatedAdoptionOfAReleasedClaimIsRefused(t *testing.T) { t.Fatalf("release: %v", err) } - assertNotAdopted(t, b, sb, "repeated adopt of a released claim") + mustNotAdopt(t, b, sb, "repeated adopt of a released claim") if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); ok { t.Errorf("row %+v outlived the released claim", rec) } @@ -1328,7 +1306,7 @@ func TestFailoverSweepDropsTheRowOfAnArchiveThatNeverCommitted(t *testing.T) { func TestFailoverSweepRearmsATombstoneItsArchivedClaimHolds(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, loseConsumeReply: true} @@ -1350,7 +1328,7 @@ func TestFailoverSweepRearmsATombstoneItsArchivedClaimHolds(t *testing.T) { func TestFailoverSweepLeavesADepartedOwnersWakeTombstone(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) epoch := c.take(t, sb.ID, "a") @@ -1365,15 +1343,13 @@ func TestFailoverSweepLeavesADepartedOwnersWakeTombstone(t *testing.T) { func TestFailoverRestartAfterAConsumeWhileAPeerSweptKeepsTheClaim(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) epoch := c.take(t, sb.ID, "a") _, _ = a.commitIfLive(t.Context(), sb, func() { sb.ArchiveEpoch = epoch }) c.consume(t, sb.ID, epoch) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") b.sweepOrphanArchives(t.Context()) c.renew(t, "a") @@ -1426,7 +1402,7 @@ func TestFailoverArchiveExportsNothingForARowAnotherNodeHolds(t *testing.T) { func TestFailoverAWakeKeepsItsFenceAgainstThisNodesOwnAdoption(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -1459,7 +1435,7 @@ func TestFailoverAWakeKeepsItsFenceAgainstThisNodesOwnAdoption(t *testing.T) { func TestFailoverConcurrentAdoptionsOnOneNodeAllAnswerForIt(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) for _, m := range []*Manager{a, b} { if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "acme", Token: "tok-acme"}); err != nil { t.Fatalf("put tenant: %v", err) @@ -1501,7 +1477,7 @@ func TestFailoverConcurrentAdoptionsOnOneNodeAllAnswerForIt(t *testing.T) { func TestFailoverACoalescedAdoptionStillNeedsTheToken(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.expire("a") @@ -1552,7 +1528,7 @@ func TestFailoverArchivePersistsItsIntentBeforeThePut(t *testing.T) { func TestFailoverRestartDropsAClaimWhoseInterruptedArchiveWasConsumedElsewhere(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) a.mu.Lock() @@ -1652,7 +1628,7 @@ func TestFailoverAnAbandonKeepsTheExportWhileItsIntentIsDurable(t *testing.T) { func TestFailoverAnOwnerReturningWithItsWakeCommittedPurgesItsTombstone(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -1660,9 +1636,7 @@ func TestFailoverAnOwnerReturningWithItsWakeCommittedPurgesItsTombstone(t *testi if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") b.sweepOrphanArchives(t.Context()) c.renew(t, "a") @@ -1679,7 +1653,7 @@ func TestFailoverAnOwnerReturningWithItsWakeCommittedPurgesItsTombstone(t *testi func TestFailoverRestartDropsAClaimConsumedWhileItSettlesItsInterruptedArchive(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) ck := interruptArchive(t, a, sb, true) @@ -1727,7 +1701,7 @@ func TestFailoverRestartUndoesItsOwnInterruptedArchiveAndItsIntent(t *testing.T) func TestFailoverReleaseAfterALostArchivePutReplyLeavesNothingAdoptable(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) c.archiveLosingPutReply(t, a, sb) @@ -1834,7 +1808,7 @@ func TestFailoverAWakeWithoutARegistryKeepsItsPendingIntent(t *testing.T) { func TestFailoverAReleaseFindsItsUnsettledArchiveTakenElsewhere(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) a.mu.Lock() @@ -1860,40 +1834,38 @@ func TestFailoverAReleaseFindsItsUnsettledArchiveTakenElsewhere(t *testing.T) { } func TestALookupRacingAWakeAdoptionTakesTheRowOnce(t *testing.T) { - c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) - sb := mustClaim(t, a, testKey) - mustArchive(t, a, sb) - rec, _, err := c.reg.Get(t.Context(), sb.ID) - if err != nil { - t.Fatalf("row: %v", err) - } - if _, err = c.reg.Release(t.Context(), sb.ID, rec.Epoch, "a"); err != nil { - t.Fatalf("release: %v", err) - } - reg := &hookRegistry{Registry: c.reg} - b.archives = reg - lookup := make(chan bool, 1) - reg.afterTake = func() { - go func() { - adopted, _ := b.AdoptArchived(t.Context(), sb.ID, sb.Token) - lookup <- adopted - }() - select { - case <-lookup: - lookup <- true - case <-time.After(200 * time.Millisecond): + synctest.Test(t, func(t *testing.T) { + c := newFailoverCluster(t) + a, b := c.pair(t) + sb := mustClaim(t, a, testKey) + mustArchive(t, a, sb) + rec, _, err := c.reg.Get(t.Context(), sb.ID) + if err != nil { + t.Fatalf("row: %v", err) } - } - if adopted, adoptErr := b.AdoptForWake(t.Context(), sb.ID, sb.Token); adoptErr != nil || !adopted { - t.Fatalf("wake adoption: %v %v", adopted, adoptErr) - } - if !<-lookup { - t.Error("the racing lookup did not find the adopted claim") - } - if got := reg.takes.Load(); got != 1 { - t.Errorf("row taken %d times, want once", got) - } + if _, err = c.reg.Release(t.Context(), sb.ID, rec.Epoch, "a"); err != nil { + t.Fatalf("release: %v", err) + } + reg := &hookRegistry{Registry: c.reg} + b.archives = reg + lookup := make(chan bool, 1) + reg.afterTake = func() { + go func() { + adopted, _ := b.AdoptArchived(t.Context(), sb.ID, sb.Token) + lookup <- adopted + }() + synctest.Wait() + } + if adopted, adoptErr := b.AdoptForWake(t.Context(), sb.ID, sb.Token); adoptErr != nil || !adopted { + t.Fatalf("wake adoption: %v %v", adopted, adoptErr) + } + if !<-lookup { + t.Error("the racing lookup did not find the adopted claim") + } + if got := reg.takes.Load(); got != 1 { + t.Errorf("row taken %d times, want once", got) + } + }) } type failoverCluster struct { @@ -1953,6 +1925,11 @@ func (c *failoverCluster) node(t *testing.T, name, dataDir string, deleteAfter i return m } +func (c *failoverCluster) pair(t *testing.T) (a, b *Manager) { + t.Helper() + return c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) +} + func (c *failoverCluster) restart(t *testing.T, old *Manager) *Manager { t.Helper() return c.nodeOn(t, fakeOf(old), old.nodeID, old.dataDir, keepAnHour) @@ -2026,79 +2003,40 @@ type hookRegistry struct { } func (r *hookRegistry) Put(ctx context.Context, rec archives.Record) (int64, error) { - if r.failPut { - return 0, archives.ErrUnavailable - } - if hook := r.beforePut; hook != nil { - r.beforePut = nil - hook() - } - epoch, err := r.Registry.Put(ctx, rec) - if r.losePutReply && err == nil { - r.losePutReply = false - return 0, archives.ErrUnavailable - } - return epoch, err + return r.write(func() (int64, error) { return r.Registry.Put(ctx, rec) }) } func (r *hookRegistry) Replace(ctx context.Context, rec archives.Record, expect int64) (int64, error) { r.replaces++ - if r.failPut { - return 0, archives.ErrUnavailable - } - if hook := r.beforePut; hook != nil { - r.beforePut = nil - hook() - } - epoch, err := r.Registry.Replace(ctx, rec, expect) - if r.losePutReply && err == nil { - r.losePutReply = false - return 0, archives.ErrUnavailable - } - return epoch, err + return r.write(func() (int64, error) { return r.Registry.Replace(ctx, rec, expect) }) } func (r *hookRegistry) Take(ctx context.Context, id string, expect int64, owner string) (int64, error) { - if hook := r.beforeTake; hook != nil { - r.beforeTake = nil - hook() - } + fire(&r.beforeTake) epoch, err := r.Registry.Take(ctx, id, expect, owner) if err == nil { r.takes.Add(1) - if hook := r.afterTake; hook != nil { - r.afterTake = nil - hook() - } + fire(&r.afterTake) } - if r.loseTakeReply && err == nil { - r.loseTakeReply = false + if err == nil && lost(&r.loseTakeReply) { return 0, archives.ErrUnavailable } return epoch, err } func (r *hookRegistry) Release(ctx context.Context, id string, expect int64, owner string) (int64, error) { - if hook := r.beforeRelease; hook != nil { - r.beforeRelease = nil - hook() - } + fire(&r.beforeRelease) released, err := r.Registry.Release(ctx, id, expect, owner) - if r.loseReleaseReply && err == nil { - r.loseReleaseReply = false + if err == nil && lost(&r.loseReleaseReply) { return 0, archives.ErrUnavailable } return released, err } func (r *hookRegistry) Delete(ctx context.Context, id string, epoch int64) error { - if hook := r.beforeDelete; hook != nil { - r.beforeDelete = nil - hook() - } + fire(&r.beforeDelete) err := r.Registry.Delete(ctx, id, epoch) - if err == nil && r.loseDeleteReply { - r.loseDeleteReply = false + if err == nil && lost(&r.loseDeleteReply) { return archives.ErrUnavailable } return err @@ -2106,26 +2044,44 @@ func (r *hookRegistry) Delete(ctx context.Context, id string, epoch int64) error func (r *hookRegistry) Rows(ctx context.Context, ids []string) (map[string]archives.Record, error) { rows, err := r.Registry.Rows(ctx, ids) - if hook := r.afterRows; hook != nil { - r.afterRows = nil - hook() - } + fire(&r.afterRows) return rows, err } func (r *hookRegistry) Consume(ctx context.Context, id string, expect int64) (int64, error) { - if hook := r.beforeConsume; hook != nil { - r.beforeConsume = nil - hook() - } + fire(&r.beforeConsume) epoch, err := r.Registry.Consume(ctx, id, expect) - if r.loseConsumeReply && err == nil { - r.loseConsumeReply = false + if err == nil && lost(&r.loseConsumeReply) { return 0, archives.ErrUnavailable } return epoch, err } +func (r *hookRegistry) write(do func() (int64, error)) (int64, error) { + if r.failPut { + return 0, archives.ErrUnavailable + } + fire(&r.beforePut) + epoch, err := do() + if err == nil && lost(&r.losePutReply) { + return 0, archives.ErrUnavailable + } + return epoch, err +} + +func fire(hook *func()) { + if h := *hook; h != nil { + *hook = nil + h() + } +} + +func lost(flag *bool) bool { + was := *flag + *flag = false + return was +} + type unreadableStore struct { store.Store } diff --git a/sandboxd/pool/golden.go b/sandboxd/pool/golden.go index ff4bd34e0..76a04a595 100644 --- a/sandboxd/pool/golden.go +++ b/sandboxd/pool/golden.go @@ -213,7 +213,6 @@ func (m *Manager) dumpGolden(ctx context.Context, name, snap, final string) erro return m.exportGolden(ctx, snap, final) } -// exportGolden exports snap into final through a unique sibling *.tmp dir. func (m *Manager) exportGolden(ctx context.Context, snap, final string) error { staging, err := os.MkdirTemp(filepath.Dir(final), filepath.Base(final)+"-*.tmp") if err != nil { diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 72a7a668d..e01fd42db 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -80,7 +80,7 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error return ErrVolumeCapture } if sb.Key.Net == types.NetEgress { - // cocoon resumes the guest before its fresh tap can be re-locked + // restore resumes the guest inside the call, before sandboxd could re-lock its tap return ErrNoEgressHibernate } if err := m.settleCapture(ctx, sb); err != nil { @@ -282,9 +282,10 @@ func (m *Manager) idleOnce(ctx context.Context) { // Hibernates are seconds-long engine snapshots, so fan out off the housekeeping loop go func() { defer m.sweeps.idle.Store(false) + logger := log.WithFunc("pool.idleOnce") m.runBounded(ctx, len(victims), func(ctx context.Context, i int) { sb := victims[i] - logSweepResult(ctx, log.WithFunc("pool.idleOnce"), m.idleHibernate(ctx, sb, now), "idle-hibernated "+sb.ID, "idle-hibernate "+sb.ID) + logSweepResult(ctx, logger, m.idleHibernate(ctx, sb, now), "idle-hibernated "+sb.ID, "idle-hibernate "+sb.ID) }).Wait() }() } diff --git a/sandboxd/pool/hibernate_test.go b/sandboxd/pool/hibernate_test.go index f206204ee..eeb78dbfc 100644 --- a/sandboxd/pool/hibernate_test.go +++ b/sandboxd/pool/hibernate_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" "testing/synctest" "time" @@ -85,12 +86,8 @@ func TestAReleaseRacingAWakeBillsTheWakeFirst(t *testing.T) { sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) m.store.writeMu.Lock() - locked := true - t.Cleanup(func() { - if locked { - m.store.writeMu.Unlock() - } - }) + unlock := sync.OnceFunc(m.store.writeMu.Unlock) + t.Cleanup(unlock) woke := make(chan error, 1) go func() { _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) @@ -111,8 +108,7 @@ func TestAReleaseRacingAWakeBillsTheWakeFirst(t *testing.T) { defer m.mu.Unlock() return m.claimed[sb.ID] == nil }) - m.store.writeMu.Unlock() - locked = false + unlock() <-woke if err := <-released; err != nil { t.Fatalf("release: %v", err) diff --git a/sandboxd/pool/intercept_test.go b/sandboxd/pool/intercept_test.go index 9e6f5c356..e57b9b649 100644 --- a/sandboxd/pool/intercept_test.go +++ b/sandboxd/pool/intercept_test.go @@ -12,38 +12,35 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestGoldenBuildInstallsCAForInterceptPool(t *testing.T) { - eng := newFakeEngine() - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: interceptPolicy()}) - if m.out.CAFingerprint() == "" { - t.Fatal("egress CA not loaded for an intercept pool") - } - final := buildGoldenAt(t, m, testKey, "") - if len(eng.caInstalls) != 1 { - t.Fatalf("InstallCACert calls = %d, want 1", len(eng.caInstalls)) - } - stamp, err := os.ReadFile(final + goldenStampSuffix) - if err != nil { - t.Fatalf("read golden stamp: %v", err) - } - if want := m.goldenStamp(m.view.Load(), testKey, ""); string(stamp) != want { - t.Errorf("stamp = %q, want %q", stamp, want) - } -} - -func TestGoldenBuildSkipsCAForPlainPool(t *testing.T) { - eng := newFakeEngine() - plain := &egress.Policy{Allow: []egress.Rule{{Host: "api.github.com", Secret: "gh"}}} - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: plain}) - if m.out.CAFingerprint() != "" { - t.Error("egress CA loaded though no pool intercepts") - } - final := buildGoldenAt(t, m, testKey, "") - if len(eng.caInstalls) != 0 { - t.Errorf("InstallCACert called %d times for a plain pool", len(eng.caInstalls)) - } - if stamp, err := os.ReadFile(final + goldenStampSuffix); err != nil || string(stamp) != m.goldenStamp(m.view.Load(), testKey, "") { - t.Errorf("stamp = %q (%v), want one without a CA fingerprint", stamp, err) +func TestGoldenBuildInstallsCAOnlyForAnInterceptPool(t *testing.T) { + tests := []struct { + name string + policy *egress.Policy + caLoaded bool + installs int + }{ + {"intercept pool", interceptPolicy(), true, 1}, + {"plain pool", plainPolicy(), false, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eng := newFakeEngine() + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: tt.policy}) + if loaded := m.out.CAFingerprint() != ""; loaded != tt.caLoaded { + t.Fatalf("egress CA loaded = %v, want %v", loaded, tt.caLoaded) + } + final := buildGoldenAt(t, m, testKey, "") + if len(eng.caInstalls) != tt.installs { + t.Fatalf("InstallCACert calls = %d, want %d", len(eng.caInstalls), tt.installs) + } + stamp, err := os.ReadFile(final + goldenStampSuffix) + if err != nil { + t.Fatalf("read golden stamp: %v", err) + } + if want := m.goldenStamp(m.view.Load(), testKey, ""); string(stamp) != want { + t.Errorf("stamp = %q, want %q", stamp, want) + } + }) } } @@ -76,7 +73,7 @@ func TestGoldenCAStampRebuildsOnMismatch(t *testing.T) { } p.goldenDir = "" editEgress(t, m, func(cfg *config.Config) { - cfg.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "api.github.com", Secret: "gh"}}}}} + cfg.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1, Egress: plainPolicy()}} }) m.adoptGolden(p, "") if p.goldenDir != "" { @@ -84,30 +81,28 @@ func TestGoldenCAStampRebuildsOnMismatch(t *testing.T) { } } -func TestColdProvisionInstallsCAForInterceptPool(t *testing.T) { - eng := newFakeEngine() - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: interceptPolicy()}) - sb, err := m.provision(t.Context(), testKey, "") - if err != nil { - t.Fatalf("cold provision: %v", err) - } - m.destroy(t.Context(), sb.VMName) - if n := len(eng.caInstalls); n != 1 { - t.Errorf("InstallCACert calls = %d, want 1 (pre-golden cold claim must trust the root)", n) - } -} - -func TestColdProvisionSkipsCAForPlainPool(t *testing.T) { - eng := newFakeEngine() - plain := &egress.Policy{Allow: []egress.Rule{{Host: "api.github.com", Secret: "gh"}}} - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: plain}) - sb, err := m.provision(t.Context(), testKey, "") - if err != nil { - t.Fatalf("cold provision: %v", err) - } - m.destroy(t.Context(), sb.VMName) - if n := len(eng.caInstalls); n != 0 { - t.Errorf("InstallCACert called %d times for a plain pool", n) +func TestColdProvisionInstallsCAOnlyForAnInterceptPool(t *testing.T) { + tests := []struct { + name string + policy *egress.Policy + installs int + }{ + {"intercept pool", interceptPolicy(), 1}, + {"plain pool", plainPolicy(), 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eng := newFakeEngine() + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: tt.policy}) + sb, err := m.provision(t.Context(), testKey, "") + if err != nil { + t.Fatalf("cold provision: %v", err) + } + m.destroy(t.Context(), sb.VMName) + if n := len(eng.caInstalls); n != tt.installs { + t.Errorf("InstallCACert calls = %d, want %d", n, tt.installs) + } + }) } } @@ -132,6 +127,10 @@ func TestInterceptPoolAllowsPromote(t *testing.T) { } } +func plainPolicy() *egress.Policy { + return &egress.Policy{Allow: []egress.Rule{{Host: "api.github.com", Secret: "gh"}}} +} + func interceptPolicy() *egress.Policy { return &egress.Policy{Allow: []egress.Rule{{Host: "api.github.com", Secret: "gh", Intercept: egress.InterceptAlways}}} } diff --git a/sandboxd/pool/interrupted_test.go b/sandboxd/pool/interrupted_test.go index 78185591a..9608f6fc9 100644 --- a/sandboxd/pool/interrupted_test.go +++ b/sandboxd/pool/interrupted_test.go @@ -305,11 +305,7 @@ func TestAFailedHibernateNeverStartsAStoppedVM(t *testing.T) { m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) eng.hibernateErr = errors.New("signal: killed") - eng.captureHook = func() { - eng.mu.Lock() - defer eng.mu.Unlock() - eng.stopped[sb.VMName] = true - } + eng.captureHook = func() { killVMM(eng, sb.VMName) } if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success") } @@ -365,9 +361,7 @@ func TestAFailedCaptureKeepsItsIntentWhileSandboxdStops(t *testing.T) { } waitStarted(t, eng, 1) waitSettled(t, sb) - if err := m.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, m) if tt.intent(journaled(t, m)[sb.ID]) == "" || sb.Failed != "" { t.Errorf("journal lost the intent or failed the claim: %q", sb.Failed) } diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index b1cbba6d8..ed68205eb 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -6,7 +6,6 @@ import ( "fmt" "net" "os" - "sync" "sync/atomic" "testing" "testing/synctest" @@ -531,17 +530,14 @@ func TestLeaseConsumeDoesNotRaceAnUnfenceRewritingTheEpoch(t *testing.T) { if err != nil { t.Fatalf("rows: %v", err) } - var unfence sync.WaitGroup - unfence.Go(func() { + a.archives = &hookRegistry{Registry: c.reg, beforeConsume: func() { if _, takeErr := a.takeOwnRows(t.Context(), map[string]*types.Sandbox{sb.ID: sb}, rows); takeErr != nil { t.Errorf("take own rows: %v", takeErr) } - }) - time.Sleep(20 * time.Millisecond) + }} sb.Transition.Lock() consumed, err := a.consumeArchive(t.Context(), sb) sb.Transition.Unlock() - unfence.Wait() if err != nil || consumed == 0 { t.Errorf("consume after the unfence took the row: epoch %d, %v", consumed, err) } diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 5ce7b0129..aadfac64d 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/cocoonstack/sandbox/sandboxd/config" - "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/outbound" "github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest" @@ -207,7 +206,7 @@ func TestWakeOfAFailedClaimRearmsEgressAfterARestart(t *testing.T) { eng.sockRoot = outboundtest.SockRoot(t) cfg := &config.Config{ DataDir: t.TempDir(), Bridges: []string{"sbxbr0"}, EgressCA: writeTestEgressCA(t), VMMRestart: types.VMMRestartNone, - Pools: []config.PoolSpec{{PoolKey: testKey, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}}}}, + Pools: []config.PoolSpec{{PoolKey: testKey, Egress: hostPolicy()}}, } first := mustManager(t, cfg, eng) sb := mustClaim(t, first, testKey) @@ -241,7 +240,7 @@ func TestWakeOfAFailedClaimRearmsEgressAfterARestart(t *testing.T) { func TestRestartKeepsAnEgressDoorThatSurvivedTheVMM(t *testing.T) { eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}}}) + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Egress: hostPolicy()}) sb := mustClaim(t, m, testKey) killVMM(eng, sb.VMName) @@ -292,7 +291,7 @@ func TestRestartThatCannotArmTheEgressDoorFailsAndSpendsTheBudget(t *testing.T) } eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}}}) + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Egress: hostPolicy()}) sb := mustClaim(t, m, testKey) killVMM(eng, sb.VMName) if err := os.Chmod(eng.sockRoot, 0o500); err != nil { diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index a26aab64d..faf7cb611 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -225,6 +225,7 @@ func (m *Manager) settleMoveIn(ctx context.Context, sb *types.Sandbox) { // False while the registry is down. func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { + logger := log.WithFunc("pool.resolveMoveIn") sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() @@ -235,7 +236,7 @@ func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { } rec, ok, err := m.moves.Get(ctx, sb.ID) if err != nil { - log.WithFunc("pool.resolveMoveIn").Warnf(ctx, "resolve moved claim %s, retrying: %v", sb.ID, err) + logger.Warnf(ctx, "resolve moved claim %s, retrying: %v", sb.ID, err) return false } toHere := ok && rec.To == m.nodeID @@ -247,14 +248,14 @@ func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { } vm, found, inspectErr := m.eng.Inspect(ctx, sb.VMName) if inspectErr != nil { - log.WithFunc("pool.resolveMoveIn").Warnf(ctx, "inspect moved claim %s, retrying: %v", sb.ID, inspectErr) + logger.Warnf(ctx, "inspect moved claim %s, retrying: %v", sb.ID, inspectErr) return false } m.startMoveIn(ctx, sb, rec.From, committed, found && vm.State != vmStateRunning) return true case toHere && (rec.State == moves.Pending || rec.State == moves.Final): if _, err = m.moves.Advance(ctx, sb.ID, rec.Epoch, rec.State, moves.Aborted); err != nil { - log.WithFunc("pool.resolveMoveIn").Warnf(ctx, "abort the move of %s, retrying: %v", sb.ID, err) + logger.Warnf(ctx, "abort the move of %s, retrying: %v", sb.ID, err) return false } } diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index ac6823574..1f6354a51 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -213,8 +213,7 @@ func (m *Manager) moveBlocker(sb *types.Sandbox) string { } func (m *Manager) journalMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) error { - journal := mv.MoveOut - live, err := m.commitIfLive(ctx, sb, func() { sb.MoveOut = &journal }) + live, err := m.commitIfLive(ctx, sb, func() { sb.MoveOut = new(mv.MoveOut) }) if !live { return ErrUnknownSandbox } diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 0936dd33f..bec653d17 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -19,7 +19,6 @@ import ( "time" "github.com/cocoonstack/sandbox/sandboxd/archives" - "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/moves" "github.com/cocoonstack/sandbox/sandboxd/moves/movestest" @@ -35,7 +34,7 @@ func TestMoveHandsARunningClaimToThePeerWithBothNodesCountingIt(t *testing.T) { vm := sb.VMName var during [][2][3]int mc.p.roundHook = func(int) error { - during = append(during, [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}) + during = append(during, mc.counts("acme")) return nil } mc.p.commitHook = mc.p.roundHook @@ -52,7 +51,7 @@ func TestMoveHandsARunningClaimToThePeerWithBothNodesCountingIt(t *testing.T) { t.Errorf("step %d counts source/target %v, want 1/1/1 on both", i, counts) } } - if got := [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}; got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { + if got := mc.counts("acme"); got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { t.Errorf("counts after the move %v, want 0/0/0 on the source and 1/1/1 on the target", got) } moved, ok := mc.t.byID(sb.ID) @@ -204,7 +203,7 @@ func TestMoveWhoseCommitAnswerIsLostResolvesThroughTheRecord(t *testing.T) { if moved, ok := mc.t.byID(sb.ID); !ok || moved.MoveIn { t.Errorf("target claim %+v %v, want it started", moved, ok) } - if got := [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}; got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { + if got := mc.counts("acme"); got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { t.Errorf("counts %v, want the claim on the target only", got) } } @@ -536,7 +535,7 @@ func TestMoveNeedsNoArchiveRegistry(t *testing.T) { if moved, ok := mc.t.byID(sb.ID); !ok || moved.MoveIn { t.Errorf("target claim %+v %v, want it started", moved, ok) } - if got := [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}; got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { + if got := mc.counts("acme"); got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { t.Errorf("counts %v, want the claim on the target only", got) } } @@ -816,18 +815,7 @@ func TestMoveFinalRoundAnswersUserCallsAtOnce(t *testing.T) { cred := Cred{Token: sb.Token} ctx := t.Context() mc.p.commitHook = func(int) error { - _, forkErr := mc.s.Fork(ctx, sb.ID, cred, 1, time.Minute, "", "") - _, ckErr := mc.s.Checkpoint(ctx, sb.ID, cred, "ck", "") - _, _, promoteErr := mc.s.Promote(ctx, sb.ID, cred, "tpl", "") - for op, err := range map[string]error{ - "fork": forkErr, "checkpoint": ckErr, "promote": promoteErr, - "hibernate": mc.s.Hibernate(ctx, sb.ID, cred), - "release": mc.s.Release(ctx, sb.ID, cred), - } { - if !errors.Is(err, ErrEvacuating) { - t.Errorf("%s during the final round: %v, want ErrEvacuating", op, err) - } - } + assertEvacuating(t, evacuatingCalls(ctx, mc.s, sb.ID, cred), "during the final round") return nil } if _, err := mc.s.Move(ctx, sb.ID, "b", time.Minute); err != nil { @@ -1147,22 +1135,12 @@ func TestMoveReceivedClaimWaitingOutAnOutageIsNotHibernatedOrServed(t *testing.T mc.t.mu.Lock() mc.t.evac = nil mc.t.mu.Unlock() - _, forkErr := mc.t.Fork(ctx, sb.ID, cred, 1, time.Minute, "", "") - _, ckErr := mc.t.Checkpoint(ctx, sb.ID, cred, "ck", "") - _, _, promoteErr := mc.t.Promote(ctx, sb.ID, cred, "tpl", "") - _, renewErr := mc.t.Renew(ctx, sb.ID, cred, time.Hour, "") - _, dialErr := mc.t.DialPort(ctx, sb.ID, cred, 80) - for op, err := range map[string]error{ - "fork": forkErr, "checkpoint": ckErr, "promote": promoteErr, "renew": renewErr, "dial": dialErr, - "hibernate": mc.t.Hibernate(ctx, sb.ID, cred), - "release": mc.t.Release(ctx, sb.ID, cred), - "env": mc.t.PatchEnv(ctx, sb.ID, types.EnvPatch{"GH_TOKEN": {Value: "x", Guest: hostOnly}}, ""), - "wake": mc.t.Wake(ctx, sb.ID, cred), - } { - if !errors.Is(err, ErrEvacuating) { - t.Errorf("%s of a received claim: %v, want ErrEvacuating", op, err) - } - } + calls := evacuatingCalls(ctx, mc.t, sb.ID, cred) + _, calls["renew"] = mc.t.Renew(ctx, sb.ID, cred, time.Hour, "") + _, calls["dial"] = mc.t.DialPort(ctx, sb.ID, cred, 80) + calls["env"] = mc.t.PatchEnv(ctx, sb.ID, types.EnvPatch{"GH_TOKEN": {Value: "x", Guest: hostOnly}}, "") + calls["wake"] = mc.t.Wake(ctx, sb.ID, cred) + assertEvacuating(t, calls, "of a received claim") if _, err := mc.t.DialPort(ctx, sb.ID, Cred{Operator: true}, 80); !errors.Is(err, ErrPaused) { t.Errorf("operator dial of a received claim: %v, want ErrPaused", err) } @@ -1472,18 +1450,7 @@ func testMoveHoldsOffCallers(t *testing.T, archived bool) { if round != 2 { return nil } - _, forkErr := mc.s.Fork(t.Context(), sb.ID, cred, 1, time.Minute, "", "") - _, ckErr := mc.s.Checkpoint(t.Context(), sb.ID, cred, "ck", "") - _, _, promoteErr := mc.s.Promote(t.Context(), sb.ID, cred, "tpl", "") - for op, err := range map[string]error{ - "fork": forkErr, "checkpoint": ckErr, "promote": promoteErr, - "hibernate": mc.s.Hibernate(t.Context(), sb.ID, cred), - "release": mc.s.Release(t.Context(), sb.ID, cred), - } { - if !errors.Is(err, ErrEvacuating) { - t.Errorf("%s during a move: %v, want ErrEvacuating", op, err) - } - } + assertEvacuating(t, evacuatingCalls(t.Context(), mc.s, sb.ID, cred), "during a move") mc.s.mu.Lock() replicable := mc.s.replicable(sb) mc.s.mu.Unlock() @@ -1541,7 +1508,7 @@ func newMoveCluster(t *testing.T) *moveCluster { func newMoveClusterWith(t *testing.T, edit func(*failoverCluster)) *moveCluster { t.Helper() c := newFailoverCluster(t) - c.egressPol = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + c.egressPol = hostPolicy() edit(c) reg := movestest.NewMem() c.moves = reg @@ -1561,6 +1528,10 @@ func (mc *moveCluster) moveNode(t *testing.T, name string) *Manager { return m } +func (mc *moveCluster) counts(tenant string) [2][3]int { + return [2][3]int{moveCounts(mc.s, tenant), moveCounts(mc.t, tenant)} +} + type fakeMovePeer struct { c *failoverCluster srv *httptest.Server @@ -1757,6 +1728,25 @@ func startsOf(m *Manager) []string { return slices.Clone(eng.starts) } +func evacuatingCalls(ctx context.Context, m *Manager, id string, cred Cred) map[string]error { + calls := map[string]error{} + _, calls["fork"] = m.Fork(ctx, id, cred, 1, time.Minute, "", "") + _, calls["checkpoint"] = m.Checkpoint(ctx, id, cred, "ck", "") + _, _, calls["promote"] = m.Promote(ctx, id, cred, "tpl", "") + calls["hibernate"] = m.Hibernate(ctx, id, cred) + calls["release"] = m.Release(ctx, id, cred) + return calls +} + +func assertEvacuating(t *testing.T, calls map[string]error, when string) { + t.Helper() + for op, err := range calls { + if !errors.Is(err, ErrEvacuating) { + t.Errorf("%s %s: %v, want ErrEvacuating", op, when, err) + } + } +} + func moveCounts(m *Manager, tenant string) [3]int { m.mu.Lock() defer m.mu.Unlock() diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 625bea2c1..431448479 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -420,6 +420,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg claimed: map[string]*types.Sandbox{}, claimedVMs: map[string]*types.Sandbox{}, pendingRemovals: map[string]pendingRemoval{}, + tunReleases: map[string]netip.Addr{}, moving: map[string]*outMove{}, incoming: map[string]*inMove{}, heldTenants: map[string]int{}, @@ -466,9 +467,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg m.cfg = cfg m.view.Store(newConfigView(cfg, secrets)) for _, spec := range cfg.Pools { - p := newPool(spec.PoolKey) - p.applySpec(spec) - m.pools[spec.PoolKey] = p + m.pools[spec.PoolKey] = newPool(spec) } ca, caErr := outbound.LoadCA(cfg) if caErr != nil { @@ -537,6 +536,7 @@ func (m *Manager) Run(ctx context.Context) { m.refillOnce(ctx) case <-reap: m.dropIdleReservations() + m.dropStaleReaders(ctx) m.retryRemovals(ctx) m.reapOnce(ctx) m.idleOnce(ctx) @@ -749,6 +749,10 @@ func tenantOwns(tenant, owner string) bool { return tenant == "" || tenant == owner } +func templateVisible(tenant, owner string) bool { + return owner == "" || tenantOwns(tenant, owner) +} + func hasVolumes(sb *types.Sandbox) bool { return len(sb.Volumes) > 0 || len(sb.VolumesWant) > 0 } diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index bc3ea3a05..5655af078 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -2289,6 +2289,50 @@ func hibernateClaim(t *testing.T, m *Manager, sb *types.Sandbox) { } } +func auditLines(t *testing.T, dir string) []string { + t.Helper() + raw, err := os.ReadFile(filepath.Join(dir, "audit.jsonl")) + if err != nil { + t.Fatalf("read audit journal: %v", err) + } + return strings.Split(strings.TrimSpace(string(raw)), "\n") +} + +func reloadWith(t *testing.T, m *Manager, edit func(*config.Config)) ReloadResult { + t.Helper() + next := *m.cfg + edit(&next) + res, err := m.ReloadConfig(t.Context(), &next) + if err != nil { + t.Fatalf("reload: %v", err) + } + return res +} + +func hostPolicy() *egress.Policy { + return &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} +} + +func pendingOf(m *Manager, name string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.pendingRemovals[name] + return ok +} + +func claimedCount(m *Manager) int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.claimed) +} + +func mustFlush(t *testing.T, m *Manager) { + t.Helper() + if err := m.FlushClaims(); err != nil { + t.Fatalf("flush: %v", err) + } +} + func mustAdopt(t *testing.T, m *Manager, sb *types.Sandbox, step string) { t.Helper() if adopted, err := m.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { @@ -2303,13 +2347,6 @@ func mustNotAdopt(t *testing.T, m *Manager, sb *types.Sandbox, step string) { } } -func assertNotAdopted(t *testing.T, m *Manager, sb *types.Sandbox, step string) { - t.Helper() - if adopted, err := m.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || adopted { - t.Errorf("%s: %v %v, want refused", step, adopted, err) - } -} - func breakStore(t *testing.T, m *Manager) { t.Helper() m.store.path = filepath.Join(t.TempDir(), "gone", "claims.json") @@ -2747,3 +2784,9 @@ func liveCatalog(t *testing.T) []config.VolumeSpec { {Name: "other", Path: writeVolumeImage(t, "other.img", "other"), Writable: true}, } } + +func disableReplication(m *Manager) { + view := *m.view.Load() + view.poolReplicate = nil + m.view.Store(&view) +} diff --git a/sandboxd/pool/poolstore.go b/sandboxd/pool/poolstore.go index b510f4555..bac0eac82 100644 --- a/sandboxd/pool/poolstore.go +++ b/sandboxd/pool/poolstore.go @@ -109,9 +109,7 @@ func (m *Manager) adoptPools(ctx context.Context) error { if err := spec.ValidateLimits(); err != nil { return fmt.Errorf("restore pool %q from %s: %w", spec.Template, from, err) } - p := newPool(spec.PoolKey) - p.applySpec(spec) - m.pools[spec.PoolKey] = p + m.pools[spec.PoolKey] = newPool(spec) } logger.Infof(ctx, "restored %d API-applied pools from %s", len(set.Pools), from) return nil diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index 5f65fdc9b..81957e89b 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "slices" "testing" + "testing/synctest" "time" "github.com/cocoonstack/sandbox/sandboxd/config" @@ -23,10 +24,7 @@ func TestPromoteThenClaimClonesFromTemplate(t *testing.T) { m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - gotKey, gotDigest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } + gotKey, gotDigest := mustPromote(t, m, parent, "tpl:x", "") if len(eng.snapSaves) != 1 || !slices.Contains(eng.snapRemoves, eng.snapSaves[0]) { t.Errorf("snapSaves=%v snapRemoves=%v, want one transient snapshot dropped", eng.snapSaves, eng.snapRemoves) } @@ -70,23 +68,14 @@ func TestRepromoteContentDigestTracksExportBytes(t *testing.T) { m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - _, firstDigest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:digest", "") - if err != nil { - t.Fatalf("first Promote: %v", err) - } - _, secondDigest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:digest", "") - if err != nil { - t.Fatalf("same-byte Promote: %v", err) - } + _, firstDigest := mustPromote(t, m, parent, "tpl:digest", "") + _, secondDigest := mustPromote(t, m, parent, "tpl:digest", "") if secondDigest != firstDigest { t.Errorf("same export bytes changed digest: %q then %q", firstDigest, secondDigest) } eng.exportContent = []byte("generation two") - thirdKey, thirdDigest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:digest", "") - if err != nil { - t.Fatalf("changed-byte Promote: %v", err) - } + thirdKey, thirdDigest := mustPromote(t, m, parent, "tpl:digest", "") if thirdDigest == firstDigest { t.Errorf("changed export bytes kept digest %q", thirdDigest) } @@ -105,9 +94,7 @@ func TestPromoteHibernatedUsesWakeImage(t *testing.T) { parent := mustClaim(t, m, testKey) hibernateClaim(t, m, parent) - if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:hib", ""); err != nil { - t.Fatalf("Promote: %v", err) - } + mustPromote(t, m, parent, "tpl:hib", "") if len(eng.snapSaves) != 0 { t.Errorf("snapSaves %v, want none — the hibernate image is the source", eng.snapSaves) } @@ -144,20 +131,14 @@ func TestTemplateDigestGuardReadsTheSharedRecord(t *testing.T) { eng.exportContent = []byte("generation one") m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - key, stale, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:shared", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } + key, stale := mustPromote(t, m, parent, "tpl:shared", "") peer := newTestManager(t, newFakeEngine()) peer.tpls = m.tpls id := store.TemplateID(key.Hash()) peer.tplSet[id] = TemplateInfo{Key: key, ContentDigest: stale} eng.exportContent = []byte("generation two") - _, latest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:shared", "") - if err != nil { - t.Fatalf("re-Promote: %v", err) - } + _, latest := mustPromote(t, m, parent, "tpl:shared", "") if err := peer.SetTemplateLabels(t.Context(), key, types.Metadata{"a": "b"}, "", stale); !errors.Is(err, ErrTemplateReplaced) { t.Errorf("labels with the replaced digest: %v, want ErrTemplateReplaced", err) } @@ -180,10 +161,7 @@ func TestDeleteTemplateRefusesPooledUnknownAndReplacedTemplates(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 0}) parent := mustClaim(t, m, testKey) - _, digest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:del", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } + _, digest := mustPromote(t, m, parent, "tpl:del", "") key := tplKey("tpl:del") if err := m.DeleteTemplate(t.Context(), testKey, "", ""); !errors.Is(err, ErrPooledTemplate) { @@ -224,9 +202,7 @@ func TestTemplateRecordLockEvictsWithTheRecord(t *testing.T) { if hasRecLock(m, id) { t.Error("recLocks kept an entry for a template that was never promoted") } - if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, key.Template, ""); err != nil { - t.Fatalf("Promote: %v", err) - } + mustPromote(t, m, parent, key.Template, "") if err := m.DeleteTemplate(t.Context(), key, "", ""); err != nil { t.Fatalf("DeleteTemplate: %v", err) } @@ -253,6 +229,23 @@ func TestReconcileSweepsGoldenTmpDirs(t *testing.T) { } } +func TestResolveGoldenNeverServesAPooledKeyFromATemplate(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + parent := mustClaim(t, m, testKey) + key, _ := mustPromote(t, m, parent, "tpl:pooled", "") + m.mu.Lock() + m.pools[key] = newPool(config.PoolSpec{PoolKey: key}) + m.mu.Unlock() + golden, err := m.resolveGolden(t.Context(), key, "") + if err != nil { + t.Fatalf("resolveGolden: %v", err) + } + golden.release() + if golden.dir != "" || golden.promoted { + t.Errorf("resolveGolden served pooled key from template %q; want cold-boot until the pool golden lands", golden.dir) + } +} + func TestResolveGoldenSkipsPromotedEgressTemplate(t *testing.T) { m := newTestManager(t, newFakeEngine()) staging := stageTemplate(t, m, store.TemplateID(egKey.Hash())) @@ -273,10 +266,7 @@ func TestTemplateTenantScopedDelete(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) parent := claimTenant(t, m, "acme") - key, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:tenant", "acme") - if err != nil { - t.Fatalf("Promote: %v", err) - } + key, _ := mustPromote(t, m, parent, "tpl:tenant", "acme") if err := m.DeleteTemplate(t.Context(), key, "beta", ""); !errors.Is(err, ErrUnknownTemplate) { t.Errorf("cross-tenant delete: %v, want ErrUnknownTemplate", err) @@ -284,9 +274,7 @@ func TestTemplateTenantScopedDelete(t *testing.T) { if err := m.DeleteTemplate(t.Context(), key, "acme", ""); err != nil { t.Errorf("own delete: %v", err) } - if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:tenant", "acme"); err != nil { - t.Fatalf("re-promote: %v", err) - } + mustPromote(t, m, parent, "tpl:tenant", "acme") if err := m.DeleteTemplate(t.Context(), key, "", ""); err != nil { t.Errorf("root delete: %v", err) } @@ -310,9 +298,7 @@ func TestPromoteFailsClosedOnMetaError(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) a := claimTenant(t, m, "acme") - if _, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "shared:v1", "acme"); err != nil { - t.Fatalf("promote: %v", err) - } + mustPromote(t, m, a, "shared:v1", "acme") key := tplKey("shared:v1") meta := filepath.Join(m.dataDir, "checkpoints", store.TemplateID(key.Hash()), store.MetaFile) if err := os.Chmod(meta, 0o000); err != nil { @@ -328,9 +314,7 @@ func TestPromoteRefusesCrossTenantOverwrite(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) a := claimTenant(t, m, "acme") - if _, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "shared:v1", "acme"); err != nil { - t.Fatalf("acme promote: %v", err) - } + mustPromote(t, m, a, "shared:v1", "acme") b := claimTenant(t, m, "beta") if _, _, err := m.Promote(t.Context(), b.ID, Cred{Token: b.Token}, "shared:v1", "beta"); !errors.Is(err, ErrTemplateOwned) { t.Errorf("beta overwrite: %v, want ErrTemplateOwned", err) @@ -346,15 +330,9 @@ func TestTemplateClaimIsTenantScoped(t *testing.T) { m := newTestManager(t, eng) a := claimTenant(t, m, "acme") - private, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") - if err != nil { - t.Fatalf("acme promote: %v", err) - } + private, _ := mustPromote(t, m, a, "acme-private", "acme") r := claimTenant(t, m, "") - shared, _, err := m.Promote(t.Context(), r.ID, Cred{Token: r.Token}, "ops-shared", "") - if err != nil { - t.Fatalf("root promote: %v", err) - } + shared, _ := mustPromote(t, m, r, "ops-shared", "") if _, err := m.ClaimProvision(t.Context(), private, ClaimOptions{RequirePromoted: true, TTL: time.Hour, Tenant: "beta"}); !errors.Is(err, ErrUnknownTemplate) { t.Errorf("beta claiming acme's template: %v, want ErrUnknownTemplate", err) @@ -383,10 +361,7 @@ func TestTemplateClaimIsTenantScoped(t *testing.T) { func TestHasPromotedTemplateIsTenantScoped(t *testing.T) { m := newTestManager(t, newFakeEngine()) a := claimTenant(t, m, "acme") - key, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") - if err != nil { - t.Fatalf("promote: %v", err) - } + key, _ := mustPromote(t, m, a, "acme-private", "acme") if m.HasPromotedTemplate(t.Context(), key, "beta") { t.Error("beta sees acme's template as a local golden") @@ -401,10 +376,7 @@ func TestHasPromotedTemplateIsTenantScoped(t *testing.T) { func TestTemplateHashesAreTenantScoped(t *testing.T) { m := tenantManager(t, t.TempDir(), tenants.TenantSpec{Name: "acme", Token: "acme-tok"}) a := claimTenant(t, m, "acme") - key, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") - if err != nil { - t.Fatalf("promote: %v", err) - } + key, _ := mustPromote(t, m, a, "acme-private", "acme") if err := m.DeleteTenant(t.Context(), "acme"); err != nil { t.Fatalf("remove tenant: %v", err) } @@ -432,9 +404,7 @@ func TestTemplateHashesSortedForMeshCompare(t *testing.T) { m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) for _, name := range []string{"tpl:a", "tpl:b", "tpl:c", "tpl:d"} { - if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, name, ""); err != nil { - t.Fatalf("Promote %s: %v", name, err) - } + mustPromote(t, m, parent, name, "") } hashes := m.TemplateHashes() if len(hashes) != 8 { @@ -450,21 +420,15 @@ func TestTemplatesListWhatTheNodeHoldsAcrossARestart(t *testing.T) { dir := t.TempDir() m := newTestManagerAt(t, eng, dir) parent := mustClaim(t, m, testKey) - keyA, digestA, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:a", "") - if err != nil { - t.Fatalf("Promote tpl:a: %v", err) - } + keyA, digestA := mustPromote(t, m, parent, "tpl:a", "") acme := claimTenant(t, m, "acme") - keyB, digestB, err := m.Promote(t.Context(), acme.ID, Cred{Token: acme.Token}, "tpl:b", "acme") - if err != nil { - t.Fatalf("Promote tpl:b: %v", err) - } + keyB, digestB := mustPromote(t, m, acme, "tpl:b", "acme") oldID := store.TemplateID(tplKey("tpl:old").Hash()) staging := stageTemplate(t, m, oldID) - if err = os.WriteFile(filepath.Join(staging, store.MetaFile), []byte(`{"id":"`+oldID+`","created_at":"2026-01-01T00:00:00Z"}`), 0o600); err != nil { + if err := os.WriteFile(filepath.Join(staging, store.MetaFile), []byte(`{"id":"`+oldID+`","created_at":"2026-01-01T00:00:00Z"}`), 0o600); err != nil { t.Fatalf("write keyless meta: %v", err) } - if _, err = storetest.PublishDigested(t.Context(), m.tpls, staging, oldID); err != nil { + if _, err := storetest.PublishDigested(t.Context(), m.tpls, staging, oldID); err != nil { t.Fatalf("publish keyless record: %v", err) } @@ -487,7 +451,7 @@ func TestTemplatesListWhatTheNodeHoldsAcrossARestart(t *testing.T) { if hashes := restarted.TemplateHashes(); len(hashes) != 6 { t.Errorf("gossip %v, want the keyless record still routed", hashes) } - if err = restarted.DeleteTemplate(t.Context(), keyA, "", ""); err != nil { + if err := restarted.DeleteTemplate(t.Context(), keyA, "", ""); err != nil { t.Fatalf("DeleteTemplate: %v", err) } if got = restarted.Templates(); len(got) != 1 || got[0].Key != keyB { @@ -504,10 +468,7 @@ func TestTemplateLabelsPersistUntilTheTemplateIsRepromotedOrDeleted(t *testing.T dir := t.TempDir() m := newTestManagerAt(t, eng, dir) parent := mustClaim(t, m, testKey) - key, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:l", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } + key, _ := mustPromote(t, m, parent, "tpl:l", "") labelsOf := func(m *Manager) types.Metadata { t.Helper() i := slices.IndexFunc(m.Templates(), func(ti TemplateInfo) bool { return ti.Key == key }) @@ -517,10 +478,10 @@ func TestTemplateLabelsPersistUntilTheTemplateIsRepromotedOrDeleted(t *testing.T return m.Templates()[i].Labels } want := types.Metadata{"v1": "sha256:aa"} - if err = m.SetTemplateLabels(t.Context(), key, want, "", "sha256:observed-elsewhere"); !errors.Is(err, ErrTemplateReplaced) || labelsOf(m) != nil { + if err := m.SetTemplateLabels(t.Context(), key, want, "", "sha256:observed-elsewhere"); !errors.Is(err, ErrTemplateReplaced) || labelsOf(m) != nil { t.Errorf("labels with another digest: %v, want ErrTemplateReplaced and no labels written", err) } - if err = m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { + if err := m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { t.Fatalf("SetTemplateLabels: %v", err) } if got := labelsOf(m); !maps.Equal(got, want) { @@ -530,38 +491,34 @@ func TestTemplateLabelsPersistUntilTheTemplateIsRepromotedOrDeleted(t *testing.T if got := labelsOf(restarted); !maps.Equal(got, want) { t.Errorf("labels after restart %v, want %v", got, want) } - if err = m.SetTemplateLabels(t.Context(), key, types.Metadata{"x": "y"}, "acme", ""); !errors.Is(err, ErrUnknownTemplate) { + if err := m.SetTemplateLabels(t.Context(), key, types.Metadata{"x": "y"}, "acme", ""); !errors.Is(err, ErrUnknownTemplate) { t.Errorf("a tenant labeling the operator's template: %v, want ErrUnknownTemplate", err) } - if err = m.SetTemplateLabels(t.Context(), tplKey("tpl:none"), want, "", ""); !errors.Is(err, ErrUnknownTemplate) { + if err := m.SetTemplateLabels(t.Context(), tplKey("tpl:none"), want, "", ""); !errors.Is(err, ErrUnknownTemplate) { t.Errorf("an unknown template: %v, want ErrUnknownTemplate", err) } - if _, _, err = m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:l", ""); err != nil { - t.Fatalf("re-promote: %v", err) - } + mustPromote(t, m, parent, "tpl:l", "") if got := labelsOf(newTestManagerAt(t, eng, dir)); got != nil { t.Errorf("labels %v survived a re-promote, want none", got) } - if err = m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { + if err := m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { t.Fatalf("SetTemplateLabels again: %v", err) } - if err = m.SetTemplateLabels(t.Context(), key, types.Metadata{}, "", ""); err != nil || labelsOf(m) != nil { + if err := m.SetTemplateLabels(t.Context(), key, types.Metadata{}, "", ""); err != nil || labelsOf(m) != nil { t.Errorf("empty labels: %v, labels %v; want them cleared", err, labelsOf(m)) } - if err = m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { + if err := m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { t.Fatalf("SetTemplateLabels before delete: %v", err) } - if err = m.DeleteTemplate(t.Context(), key, "", ""); err != nil { + if err := m.DeleteTemplate(t.Context(), key, "", ""); err != nil { t.Fatalf("DeleteTemplate: %v", err) } - if _, _, err = m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:l", ""); err != nil { - t.Fatalf("promote after delete: %v", err) - } + mustPromote(t, m, parent, "tpl:l", "") if got := labelsOf(newTestManagerAt(t, eng, dir)); got != nil { t.Errorf("labels %v outlived the deleted template", got) } pooled := newTestManagerAt(t, eng, t.TempDir(), config.PoolSpec{PoolKey: key}) - if err = pooled.SetTemplateLabels(t.Context(), key, want, "", ""); !errors.Is(err, ErrPooledTemplate) { + if err := pooled.SetTemplateLabels(t.Context(), key, want, "", ""); !errors.Is(err, ErrPooledTemplate) { t.Errorf("a pooled key: %v, want ErrPooledTemplate", err) } } @@ -581,40 +538,40 @@ func TestRejectedLabelWritesLeakNoRecLock(t *testing.T) { } func TestClaimFromATemplateRunsBesideItsRepromote(t *testing.T) { - m := newTestManager(t, newFakeEngine()) - parent := mustClaim(t, m, testKey) - key, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } - slow := &slowPrepareStore{Store: m.tpls, entered: make(chan struct{}), release: make(chan struct{})} - m.tpls = slow - repromoted := make(chan error, 1) - go func() { - _, _, promoteErr := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") - repromoted <- promoteErr - }() - <-slow.entered - - ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) - defer cancel() - claimed := make(chan error, 1) - go func() { - _, claimErr := claimAny(ctx, m, key, ClaimOptions{}) - claimed <- claimErr - }() - select { - case err = <-claimed: - if err != nil { - t.Errorf("claim during a re-promote: %v, want a clone of the committed template", err) + synctest.Test(t, func(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + parent := mustClaim(t, m, testKey) + key, _ := mustPromote(t, m, parent, "tpl:x", "") + slow := &slowPrepareStore{Store: m.tpls, entered: make(chan struct{}), release: make(chan struct{})} + m.tpls = slow + repromoted := make(chan error, 1) + go func() { + _, _, promoteErr := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") + repromoted <- promoteErr + }() + <-slow.entered + + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + claimed := make(chan error, 1) + go func() { + _, claimErr := claimAny(ctx, m, key, ClaimOptions{}) + claimed <- claimErr + }() + synctest.Wait() + select { + case err := <-claimed: + if err != nil { + t.Errorf("claim during a re-promote: %v, want a clone of the committed template", err) + } + default: + t.Error("a claim with a 100 ms context was still blocked during a re-promote") } - case <-time.After(2 * time.Second): - t.Error("a claim with a 100 ms context was still blocked 2 s into a re-promote") - } - close(slow.release) - if err = <-repromoted; err != nil { - t.Fatalf("re-promote: %v", err) - } + close(slow.release) + if err := <-repromoted; err != nil { + t.Fatalf("re-promote: %v", err) + } + }) } type slowPrepareStore struct { @@ -629,6 +586,15 @@ func (s *slowPrepareStore) PrepareDigested(ctx context.Context, staging, id, lin return s.Store.PrepareDigested(ctx, staging, id, lineage) } +func mustPromote(t *testing.T, m *Manager, sb *types.Sandbox, name, tenant string) (types.PoolKey, string) { + t.Helper() + key, digest, err := m.Promote(t.Context(), sb.ID, Cred{Token: sb.Token}, name, tenant) + if err != nil { + t.Fatalf("promote %s: %v", name, err) + } + return key, digest +} + func stageTemplate(t *testing.T, m *Manager, id string) string { t.Helper() staging, err := m.tpls.Stage(id) diff --git a/sandboxd/pool/provision.go b/sandboxd/pool/provision.go index 72cc78dfa..841545284 100644 --- a/sandboxd/pool/provision.go +++ b/sandboxd/pool/provision.go @@ -36,7 +36,7 @@ func (m *Manager) provisionResolved(ctx context.Context, key types.PoolKey, g go func (m *Manager) noteClaimClone(ctx context.Context, g goldenResolution, err error) { capReason := engine.CapacitySignature(err) m.mu.Lock() - g.held.countClone(ctx, g.gen, err == nil, capReason) + g.held.countClone(ctx, g.gen, err, capReason) retired := m.retireUnclonableLocked(g.held, g.gen, time.Now()) m.mu.Unlock() if retired { diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go index c2d3df582..22f4a0887 100644 --- a/sandboxd/pool/reader.go +++ b/sandboxd/pool/reader.go @@ -6,9 +6,11 @@ import ( "encoding/json/v2" "errors" "fmt" + "maps" "net" "os" "path/filepath" + "slices" "sync" "time" @@ -136,6 +138,22 @@ func (m *Manager) dropReader(ctx context.Context, r *reader, force bool) { m.destroyReader(ctx, r) } +func (m *Manager) dropStaleReaders(ctx context.Context) { + rs := m.readers + rs.mu.Lock() + live := slices.Collect(maps.Values(rs.live)) + rs.mu.Unlock() + for _, r := range live { + m.mu.Lock() + sb := m.claimed[r.claim] + stale := sb == nil || sleepSource(sb) != r.source + m.mu.Unlock() + if stale { + m.dropReader(ctx, r, true) + } + } +} + func (m *Manager) ensureReader(ctx context.Context, sb *types.Sandbox) error { m.mu.Lock() want := sleepSource(sb) diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index 00c8e6b76..bfebed0cd 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -166,18 +166,13 @@ func TestReaderServesOnlyPoolsThatAskForIt(t *testing.T) { func TestReloadTurnsTheReaderOnForLaterSleeps(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) - next := *m.cfg - next.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1, Reader: true}} - res, err := m.ReloadConfig(t.Context(), &next) - if err != nil { - t.Fatalf("reload: %v", err) - } + res := reloadWith(t, m, func(c *config.Config) { c.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1, Reader: true}} }) if want := []string{"pools[rt:24.04 none small].reader"}; !slices.Equal(res.Changed, want) { t.Errorf("changed %v, want %v", res.Changed, want) } sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) - if _, _, _, err = m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { + if _, _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { t.Errorf("WakeAgentSocketOrRead after the reload = %v, want ErrSleeping", err) } } @@ -649,6 +644,33 @@ func TestReaderOfAStaleSleepIsRetiredEvenWithAConnectionOpen(t *testing.T) { } } +func TestReaderWithAConnectionOpenIsRetiredOnceItsClaimWakesOrLeaves(t *testing.T) { + for _, leave := range []bool{false, true} { + eng := newFakeEngine() + m := newReaderManager(t, eng) + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") + open, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader: %v", err) + } + if leave { + err = m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) + } else { + _, _, _, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + } + if err != nil { + t.Fatalf("leave=%v: %v", leave, err) + } + m.dropStaleReaders(t.Context()) + if removes, _ := readerLifecycle(eng); !slices.Contains(removes, "sbx-reader-1") { + t.Errorf("leave=%v: removes=%v, want the reader destroyed although its connection is open", leave, removes) + } + _ = open.Close() + } +} + func TestFailedPreHibernateSyncSendsReadsToTheWakePath(t *testing.T) { for _, tt := range []struct { name string diff --git a/sandboxd/pool/reap.go b/sandboxd/pool/reap.go index 9ce89de44..e2eea3214 100644 --- a/sandboxd/pool/reap.go +++ b/sandboxd/pool/reap.go @@ -4,7 +4,6 @@ import ( "cmp" "context" "net/netip" - "slices" "time" "github.com/projecteru2/core/log" @@ -80,17 +79,14 @@ func (m *Manager) reapOnce(ctx context.Context) { logger := log.WithFunc("pool.reapOnce") failed := m.ckMarks.markAll(purges) - expired = slices.DeleteFunc(expired, func(v victim) bool { - markErr, bad := failed[v.ck] - if bad { - logger.Errorf(ctx, markErr, "mark archive ck %s; keeping %s", v.ck, v.id) - } - return bad - }) m.mu.Lock() keep := expired[:0] var purgeCks, purged []string for _, v := range expired { + if markErr, bad := failed[v.ck]; bad { + logger.Errorf(ctx, markErr, "mark archive ck %s; keeping %s", v.ck, v.id) + continue + } if v.action == reapPurge { if m.claimed[v.id] != v.sb || v.sb.ArchiveCk != v.ck { continue diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index f98fa1c26..d65cb5dcf 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -221,7 +221,6 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec return removed } -// removeStaleVM reclaims one unowned VM, reporting whether it is gone. func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool { logger := log.WithFunc("pool.removeStaleVM") if rec.State == vmStateCreating { diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index 56335f63a..493a45194 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -72,9 +72,7 @@ func TestReconcileStaleCreateSweep(t *testing.T) { if eng.removed("sbx-stale-1") != tt.wantRemove { t.Errorf("removed=%v, want %v", eng.removed("sbx-stale-1"), tt.wantRemove) } - m.mu.Lock() - _, pending := m.pendingRemovals["sbx-stale-1"] - m.mu.Unlock() + pending := pendingOf(m, "sbx-stale-1") if pending != tt.wantPending { t.Errorf("pending=%v, want %v", pending, tt.wantPending) } @@ -109,9 +107,7 @@ func TestBusyStaleCreateRetryConverges(t *testing.T) { eng.mu.Lock() _, present := eng.vms["sbx-stale-1"] eng.mu.Unlock() - m.mu.Lock() - _, pending := m.pendingRemovals["sbx-stale-1"] - m.mu.Unlock() + pending := pendingOf(m, "sbx-stale-1") removed := eng.removed("sbx-stale-1") if present != tt.wantPresent || removed != tt.wantRemove || pending != tt.wantPending { t.Errorf("present=%v removed=%v pending=%v, want %v %v %v", diff --git a/sandboxd/pool/refill.go b/sandboxd/pool/refill.go index 14d4bbd5f..9c86abe7f 100644 --- a/sandboxd/pool/refill.go +++ b/sandboxd/pool/refill.go @@ -84,7 +84,7 @@ func (m *Manager) refillOne(ctx context.Context, p *pool, golden string, gen uin sb, err := m.startVM(ctx, p.key, func(name string) (types.VMRecord, error) { return m.eng.Clone(ctx, golden, name, p.key) }) - cloned := err == nil + cloneErr := err <-m.refillSem if err == nil { if ctx.Err() == nil { @@ -112,7 +112,7 @@ func (m *Manager) refillOne(ctx context.Context, p *pool, golden string, gen uin } // A canceled context or a capacity failure says nothing about this pool's health backedOff := false - if p.countClone(ctx, gen, cloned, capReason) { + if p.countClone(ctx, gen, cloneErr, capReason) { backedOff = p.noteRefillResult(now, err != nil) } if backedOff { diff --git a/sandboxd/pool/refillbackoff_test.go b/sandboxd/pool/refillbackoff_test.go index 22e41b612..e0197104d 100644 --- a/sandboxd/pool/refillbackoff_test.go +++ b/sandboxd/pool/refillbackoff_test.go @@ -3,12 +3,14 @@ package pool import ( "context" "errors" + "fmt" "os" "testing" "testing/synctest" "time" "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" ) func TestRefillBacksOffOnlyAfterAnUnbrokenRunOfFailures(t *testing.T) { @@ -255,6 +257,23 @@ func TestRetiringAGoldenLeavesItsDirectoryForAClaimMidClone(t *testing.T) { } } +func TestClaimClonesCutOffByShutdownNeverRetireTheGolden(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + eng.cloneErr = fmt.Errorf("clone: %w", engine.ErrStopping) + m, p, g := adoptedGoldenNoWarm(t, eng) + + for i := range refillFailStreak { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err == nil { + t.Fatalf("claim %d on a stopping engine succeeded", i) + } + } + if goldenOf(m, p) != g { + t.Fatalf("golden %q after %d clones cut off by shutdown, want %q kept", goldenOf(m, p), refillFailStreak, g) + } + }) +} + func TestClaimClonesThatFailRetireTheGoldenOfAPoolWithNoRefill(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() @@ -398,39 +417,6 @@ func TestALateClaimCloneResultOfARetiredGoldenDoesNotCount(t *testing.T) { } } -func TestAFailingCloneOfAPromotedTemplateDoesNotCount(t *testing.T) { - eng := newFakeEngine() - m := envManager(t, eng, t.TempDir()) - src, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) - if err != nil { - t.Fatalf("source claim: %v", err) - } - key, _, err := m.Promote(t.Context(), src.ID, Cred{Token: src.Token}, "tpl:broken", "") - if err != nil { - t.Fatalf("promote: %v", err) - } - m.mu.Lock() - p := newPool(key) - m.pools[key] = p - m.mu.Unlock() - eng.mu.Lock() - eng.cloneErr = errors.New(`vm.restore -> 500 "Memory manager error"`) - eng.mu.Unlock() - clones := eng.cloneCount() - - for range refillFailStreak { - if _, err := m.ClaimProvision(t.Context(), key, ClaimOptions{}); err == nil { - t.Fatal("claim on a failing template clone succeeded") - } - } - if n := eng.cloneCount() - clones; n != refillFailStreak { - t.Fatalf("clones %d, want each claim to clone the template", n) - } - if fails := cloneFailsOf(m, p); fails != 0 { - t.Errorf("clone fails %d on the template's pool, want 0: no pool golden was cloned", fails) - } -} - func adoptedGoldenNoWarm(t *testing.T, eng *fakeEngine) (*Manager, *pool, string) { t.Helper() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey}) diff --git a/sandboxd/pool/reload_test.go b/sandboxd/pool/reload_test.go index c96639838..c17a7c77a 100644 --- a/sandboxd/pool/reload_test.go +++ b/sandboxd/pool/reload_test.go @@ -16,6 +16,7 @@ import ( "sync" "sync/atomic" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/egress" @@ -30,13 +31,10 @@ var interceptOnKey = types.PoolKey{Template: "mitm:1", Net: types.NetNone, Size: func TestReloadGivesANewKeyItsSettingsBeforeItsGolden(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) - next := *m.cfg argv := []string{"node", "-e", "0"} - next.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 3, Warmup: argv, Storage: "20G"}} - res, err := m.ReloadConfig(t.Context(), &next) - if err != nil { - t.Fatalf("reload: %v", err) - } + res := reloadWith(t, m, func(c *config.Config) { + c.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 3, Warmup: argv, Storage: "20G"}} + }) label := "pools[rt:24.04 none small]" if want := []string{label + ".warmup", label + ".storage"}; !slices.Equal(res.Changed, want) { t.Errorf("changed %v, want %v", res.Changed, want) @@ -67,11 +65,7 @@ func TestReloadRetiresAGoldenWhoseStampChangedAndLeavesClaims(t *testing.T) { waitFor(t, func() bool { return refilledTo(t, m, testKey, 2) }) sb := mustClaim(t, m, testKey) vm := sb.VMName - next := *m.cfg - next.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 2, Storage: "30G"}} - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { c.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 2, Storage: "30G"}} }) m.mu.Lock() golden, warm, live := m.pools[testKey].goldenDir, len(m.pools[testKey].warm), m.claimed[sb.ID] m.mu.Unlock() @@ -128,7 +122,7 @@ func TestReloadTurnsInterceptOnOnlyForAKeyItHasNotServedWithTheCALoaded(t *testi func TestReloadRefusesDroppingAPolicyLiveClaimsUse(t *testing.T) { eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}}}) + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: hostPolicy()}) if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{NoEgress: true}); err != nil { t.Fatalf("claim without egress: %v", err) } @@ -158,7 +152,7 @@ func TestARestartWithoutAClaimsPoolPolicyCountsTheClaimDenied(t *testing.T) { eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) dataDir := t.TempDir() - policy := &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + policy := hostPolicy() m := egressManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: policy}) mustClaim(t, m, testKey) m2 := egressManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: otherKey, Warm: 1, Egress: policy}) @@ -248,11 +242,7 @@ func TestAReloadDropsTheLiveProxiesPooledConnections(t *testing.T) { inflight := make(chan error, 1) go func() { inflight <- get("/slow") }() <-started - next := *m.cfg - next.EgressInternalAllow = []string{"10.8.0.0/16"} - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { c.EgressInternalAllow = []string{"10.8.0.0/16"} }) if err := get("/"); err != nil { t.Fatalf("get after the reload: %v", err) } @@ -293,11 +283,7 @@ func TestReloadIsAudited(t *testing.T) { func TestAReloadOfTheEgressLayerChangesTheClusterDigest(t *testing.T) { m := newTestManager(t, newFakeEngine()) before := m.ClusterDigest() - next := *m.cfg - next.EgressInternalAllow = []string{"10.8.0.0/16"} - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { c.EgressInternalAllow = []string{"10.8.0.0/16"} }) if m.ClusterDigest() == before { t.Error("the digest still covers the boot config after an egress reload") } @@ -334,14 +320,10 @@ func TestAnInterruptedReloadStillDestroysTheRetiredWarmVMs(t *testing.T) { func TestAReloadThatUnguardsTheNodeClosesAWarmVMsDoorsAtItsClaim(t *testing.T) { eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}}}) + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: hostPolicy()}) warm := refillWarmVM(t, m) pin := pinDoor(t, engine.EgressSocketPath(warm.VsockSocket)) - next := *m.cfg - next.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1}} - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { c.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1}} }) conn, err := net.Dial("unix", pin) if err != nil { t.Fatalf("the unclaimed warm VM's door: %v, want it bound until the claim", err) @@ -363,3 +345,25 @@ func TestAReloadThatUnguardsTheNodeClosesAWarmVMsDoorsAtItsClaim(t *testing.T) { t.Errorf("claim route %q, want none", route) } } + +func TestAReloadAppliesTheNodeSettings(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + reloadWith(t, m, func(c *config.Config) { + c.MaxForkCount, c.ReleaseDelaySeconds, c.VMMRestart = 1, 5, types.VMMRestartNone + c.SyncClaims, c.CheckpointTTLHours = true, 24 + c.IdleHibernateSeconds, c.ArchiveAfterSeconds, c.ArchiveDeleteAfterSeconds = 60, 120, 3600 + }) + v := m.view.Load() + got := []time.Duration{v.releaseDelay, v.ckptTTL, v.idleDefault, v.archiveAfterDefault, v.archiveDeleteDefault} + want := []time.Duration{5 * time.Second, 24 * time.Hour, time.Minute, 2 * time.Minute, time.Hour} + if !slices.Equal(got, want) || v.maxFork != 1 || v.vmmRestart != types.VMMRestartNone { + t.Errorf("durations %v max fork %d restart %q, want %v, 1 and none", got, v.maxFork, v.vmmRestart, want) + } + if !m.store.sync { + t.Error("sync_claims did not reach the store") + } + parent := mustClaim(t, m, testKey) + if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 2, 0, "", ""); !errors.Is(err, ErrBadCount) { + t.Errorf("fork of 2 over a reloaded limit of 1: %v, want ErrBadCount", err) + } +} diff --git a/sandboxd/pool/remove_test.go b/sandboxd/pool/remove_test.go index 3fb30e64f..450bd771a 100644 --- a/sandboxd/pool/remove_test.go +++ b/sandboxd/pool/remove_test.go @@ -44,17 +44,13 @@ func TestRemovalRetryConvergesConfirmedSurvivor(t *testing.T) { eng.removeErrFor = "survivor" m.destroy(t.Context(), "survivor") - m.mu.Lock() - _, pending := m.pendingRemovals["survivor"] - m.mu.Unlock() + pending := pendingOf(m, "survivor") if !pending { t.Fatal("surviving VM was not retained for retry") } m.retryRemovals(t.Context()).Wait() - m.mu.Lock() - _, pending = m.pendingRemovals["survivor"] - m.mu.Unlock() + pending = pendingOf(m, "survivor") if !pending { t.Fatal("failed retry dropped the survivor from the queue") } @@ -63,9 +59,7 @@ func TestRemovalRetryConvergesConfirmedSurvivor(t *testing.T) { eng.removeErrFor = "" eng.mu.Unlock() m.retryRemovals(t.Context()).Wait() - m.mu.Lock() - _, pending = m.pendingRemovals["survivor"] - m.mu.Unlock() + pending = pendingOf(m, "survivor") if pending || !eng.removed("survivor") { t.Fatalf("retry did not converge: pending=%v removed=%v", pending, eng.removed("survivor")) } @@ -93,9 +87,7 @@ func TestRemovalRetryFinishesEgressCleanup(t *testing.T) { eng.removeErrFor = "" eng.mu.Unlock() m.retryRemovals(t.Context()).Wait() - m.mu.Lock() - _, pending := m.pendingRemovals["survivor"] - m.mu.Unlock() + pending := pendingOf(m, "survivor") if unlocked := taps.Unlocked(); pending || !slices.Equal(unlocked, []string{"tap-survivor"}) || !eng.removed("survivor") { t.Fatalf("egress cleanup incomplete: pending=%v unlocked=%v removed=%v", pending, unlocked, eng.removed("survivor")) @@ -126,9 +118,7 @@ func TestStaleRemovalRetryKeepsTapForCleanup(t *testing.T) { eng.removeErrFor = "" eng.mu.Unlock() m.retryRemovals(t.Context()).Wait() - m.mu.Lock() - _, pending := m.pendingRemovals["sbx-stale"] - m.mu.Unlock() + pending := pendingOf(m, "sbx-stale") if pending || !eng.removed("sbx-stale") { t.Fatalf("stale retry did not converge: pending=%v removed=%v", pending, eng.removed("sbx-stale")) } diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 8f19098e8..0893bc129 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -10,7 +10,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/archives" "github.com/cocoonstack/sandbox/sandboxd/config" - "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest" "github.com/cocoonstack/sandbox/sandboxd/store" @@ -193,9 +192,7 @@ func TestReplicaRestartSettlesALostPutReply(t *testing.T) { a.mu.Lock() pending := sb.ReplicaPending a.mu.Unlock() - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) restarted := c.restartReconciled(t, a) held, ok := restarted.byID(sb.ID) @@ -795,9 +792,7 @@ func TestReplicaRestartKeepsItsRowAndStaysFencedUntilALeaseRenews(t *testing.T) sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) ck := a.replicaOf(sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) before, _, _ := c.reg.Get(t.Context(), sb.ID) restarted := c.restartReconciled(t, a) @@ -821,14 +816,12 @@ func TestReplicaRestartAdoptsTheRowsCheckpointOfAnInterruptedRound(t *testing.T) sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) first := a.replicaOf(sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) interrupted, _, publishErr := a.publishCheckpoint(t.Context(), sb, store.CheckpointID(randHex(8)), "replica", sb.Tenant, "", true) if publishErr != nil { t.Fatalf("publish: %v", publishErr) } - if _, putErr := a.putReplica(t.Context(), sb, interrupted.ID); putErr != nil { + if _, putErr := a.putRow(t.Context(), sb, interrupted.ID, true, false); putErr != nil { t.Fatalf("put: %v", putErr) } second := interrupted.ID @@ -906,9 +899,7 @@ func TestReconcileWithTheRegistryDownRecordsOnlyTheEndNoOtherNodeCanHold(t *test a := c.node(t, "a", t.TempDir(), keepAnHour) replicated, plain := mustClaim(t, a, testKey), mustClaim(t, a, plainKey) replicateNow(t, a, replicated) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) eng := fakeOf(a) delete(eng.vms, replicated.VMName) delete(eng.vms, plain.VMName) @@ -930,9 +921,7 @@ func TestReplicaRestartDropsAClaimAdoptedElsewhere(t *testing.T) { sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) ck := a.replicaOf(sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") @@ -952,9 +941,7 @@ func TestReplicaRestartDropsAClaimTheAdopterWokeAndFinished(t *testing.T) { a, b := c.node(t, "a", dirA, keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { @@ -1075,9 +1062,7 @@ func TestReplicaRestartKeepsTheCheckpointOfARoundItCannotSettleYet(t *testing.T) a.mu.Lock() pending := sb.ReplicaPending a.mu.Unlock() - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) restarted := c.restart(t, a) restarted.ckpts = &unreadableStore{Store: restarted.ckpts} @@ -1342,7 +1327,7 @@ func TestReplicaRestartMidArchiveKeepsTheClaim(t *testing.T) { func TestARunningClaimThatMovedClosesItsDoors(t *testing.T) { c := newReplicaCluster(t) - c.egressPol = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + c.egressPol = hostPolicy() ea, eb := newFakeEngine(), newFakeEngine() ea.sockRoot, eb.sockRoot = outboundtest.SockRoot(t), outboundtest.SockRoot(t) a := c.nodeOn(t, ea, "a", t.TempDir(), keepAnHour) @@ -1371,9 +1356,7 @@ func TestARunningClaimThatMovedClosesItsDoors(t *testing.T) { func restartWithoutVM(t *testing.T, c *failoverCluster, a *Manager, sb *types.Sandbox, deleteAfter int) *Manager { t.Helper() - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) eng := fakeOf(a) delete(eng.vms, sb.VMName) restarted := c.nodeOn(t, eng, a.nodeID, a.dataDir, deleteAfter) diff --git a/sandboxd/pool/running_test.go b/sandboxd/pool/running_test.go index c20da302b..c074214c3 100644 --- a/sandboxd/pool/running_test.go +++ b/sandboxd/pool/running_test.go @@ -3,7 +3,6 @@ package pool import ( "context" "errors" - "slices" "strings" "sync" "sync/atomic" @@ -222,30 +221,6 @@ func TestAReloadAppliesTheClaimCapsWithoutStoppingAClaim(t *testing.T) { } } -func TestAReloadAppliesTheNodeSettings(t *testing.T) { - m := newTestManager(t, newFakeEngine()) - next := *m.cfg - next.MaxForkCount, next.ReleaseDelaySeconds, next.VMMRestart = 1, 5, types.VMMRestartNone - next.SyncClaims, next.CheckpointTTLHours = true, 24 - next.IdleHibernateSeconds, next.ArchiveAfterSeconds, next.ArchiveDeleteAfterSeconds = 60, 120, 3600 - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } - v := m.view.Load() - got := []time.Duration{v.releaseDelay, v.ckptTTL, v.idleDefault, v.archiveAfterDefault, v.archiveDeleteDefault} - want := []time.Duration{5 * time.Second, 24 * time.Hour, time.Minute, 2 * time.Minute, time.Hour} - if !slices.Equal(got, want) || v.maxFork != 1 || v.vmmRestart != types.VMMRestartNone { - t.Errorf("durations %v max fork %d restart %q, want %v, 1 and none", got, v.maxFork, v.vmmRestart, want) - } - if !m.store.sync { - t.Error("sync_claims did not reach the store") - } - parent := mustClaim(t, m, testKey) - if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 2, 0, "", ""); !errors.Is(err, ErrBadCount) { - t.Errorf("fork of 2 over a reloaded limit of 1: %v, want ErrBadCount", err) - } -} - func TestARefusedWakeIsCountedByTheLimitThatRefusedIt(t *testing.T) { m := newTestManager(t, newFakeEngine()) editView(m, func(v *configView) { v.maxRunning = 1 }) @@ -353,6 +328,24 @@ func TestOneHandOffRunsAtATimeAndItsCooldownStartsWhenItFails(t *testing.T) { }) } +func TestQuotaRefusesClaimsPastCap(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 2}) + editView(m, func(v *configView) { v.maxClaims = 1 }) + + first := mustClaim(t, m, testKey) + if _, err := claimAny(t.Context(), m, testKey, ClaimOptions{TTL: time.Hour}); !errors.Is(err, ErrQuota) { + t.Fatalf("claim past cap: %v, want ErrQuota", err) + } + + if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { + t.Fatalf("Release: %v", err) + } + if _, err := claimAny(t.Context(), m, testKey, ClaimOptions{TTL: time.Hour}); err != nil { + t.Errorf("claim after release: %v", err) + } +} + func wakeClaim(t *testing.T, m *Manager, sb *types.Sandbox) error { t.Helper() _, _, done, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) diff --git a/sandboxd/pool/setpools.go b/sandboxd/pool/setpools.go index 13363e408..f6b746856 100644 --- a/sandboxd/pool/setpools.go +++ b/sandboxd/pool/setpools.go @@ -78,16 +78,13 @@ func (m *Manager) applyPools(ctx context.Context, desired map[types.PoolKey]conf if p := m.pools[key]; p != nil { continue } - p := newPool(key) - p.applySpec(spec) - m.pools[key] = p + m.pools[key] = newPool(spec) } m.mu.Unlock() runCtx := context.WithoutCancel(ctx) m.destroyAll(runCtx, trim).Wait() m.refillOnce(runCtx) - // Persist the applied set so a restart rebuilds from it, not the config seed. return m.poolStore.commit(seq, poolset.Set{ConfigSeed: m.configSeedHash, Pools: slices.Collect(maps.Values(desired))}) } diff --git a/sandboxd/pool/stats.go b/sandboxd/pool/stats.go index 1b41ee2fa..90b87d460 100644 --- a/sandboxd/pool/stats.go +++ b/sandboxd/pool/stats.go @@ -47,7 +47,6 @@ func (m *Manager) Stats(ctx context.Context, id string) (SandboxStats, bool) { return st, true } -// vmResidentBytes reports the resident set of the hypervisor process serving vmName. func (m *Manager) vmResidentBytes(ctx context.Context, vmName string) (int64, bool) { if vmName == "" { return 0, false @@ -56,11 +55,11 @@ func (m *Manager) vmResidentBytes(ctx context.Context, vmName string) (int64, bo if err != nil || !ok || vm.PID <= 0 { return 0, false } - return residentBytes(vm.PID) + return residentBytes(m.procDir, vm.PID) } -func residentBytes(pid int) (int64, bool) { - b, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "statm")) +func residentBytes(proc string, pid int) (int64, bool) { + b, err := os.ReadFile(filepath.Join(proc, strconv.Itoa(pid), "statm")) //nolint:gosec // an engine pid under the proc root if err != nil { return 0, false } diff --git a/sandboxd/pool/telemetry.go b/sandboxd/pool/telemetry.go index c339348e5..b9b3b5294 100644 --- a/sandboxd/pool/telemetry.go +++ b/sandboxd/pool/telemetry.go @@ -160,6 +160,12 @@ type tunFlowFrame struct { Profile string `json:"profile"` } +type stamped[F any] struct { + Time time.Time `json:"t"` + ID string `json:"id"` + Frame F `json:",embed"` +} + func (m *Manager) Counters() Counters { c := &m.counters warm, clone, cold := c.claimWarmLatency.snapshot(), c.claimCloneLatency.snapshot(), c.claimColdLatency.snapshot() @@ -225,11 +231,7 @@ func (m *Manager) recordAudit(ctx context.Context, id string, frame auditFrame) if m.audit == nil { return } - m.appendAudit(ctx, struct { - Time time.Time `json:"t"` - ID string `json:"id"` - auditFrame - }{time.Now(), id, frame}) + m.appendAudit(ctx, stamped[auditFrame]{time.Now(), id, frame}) } func (m *Manager) appendAudit(ctx context.Context, event any) { @@ -247,15 +249,10 @@ func (m *Manager) recordEgress(ctx context.Context, id, tenant string, ev egress decision = "rate_limited" } if m.audit != nil { - m.appendAudit(ctx, struct { - Time time.Time `json:"t"` - ID string `json:"id"` - egressFrame - }{ - Time: time.Now(), ID: id, + m.appendAudit(ctx, stamped[egressFrame]{time.Now(), id, egressFrame{ Op: egressEvent, Dest: ev.Host, Method: ev.Method, Port: ev.Port, Decision: decision, Secret: ev.Injected, Upstream: ev.Upstream, Tenant: tenant, Src: ev.Local, Dst: ev.Remote, - }) + }}) } m.recordUsage(ctx, usage.Event{Event: egressEvent, ID: id, Tenant: tenant, Reference: ev.Host, Upstream: ev.Upstream, Secret: ev.Injected}) } @@ -264,11 +261,7 @@ func (m *Manager) recordDNS(ctx context.Context, id, tenant string, ev outbound. if m.audit == nil { return } - m.appendAudit(ctx, struct { - Time time.Time `json:"t"` - ID string `json:"id"` - dnsFrame - }{time.Now(), id, dnsFrame{Op: dnsEvent, Name: ev.Name, Type: ev.Type, Answers: ev.Answers, Decision: ev.Decision, Tenant: tenant}}) + m.appendAudit(ctx, stamped[dnsFrame]{time.Now(), id, dnsFrame{Op: dnsEvent, Name: ev.Name, Type: ev.Type, Answers: ev.Answers, Decision: ev.Decision, Tenant: tenant}}) } func (m *Manager) recordTunFlow(ctx context.Context, id, tenant, profile string, r tunnet.FlowRecord) { @@ -279,15 +272,10 @@ func (m *Manager) recordTunFlow(ctx context.Context, id, tenant, profile string, if r.Allowed { decision = "allow" } - m.appendAudit(ctx, struct { - Time time.Time `json:"t"` - ID string `json:"id"` - tunFlowFrame - }{ - Time: r.End, ID: id, + m.appendAudit(ctx, stamped[tunFlowFrame]{r.End, id, tunFlowFrame{ Op: tunFlowEvent, Path: tunFlowPath, Tenant: tenant, Proto: protoName(r.Proto), Dst: netip.AddrPortFrom(r.Dst, r.Port), Decision: decision, Sent: r.Up, Received: r.Down, Start: r.Start, End: r.End, Profile: cmp.Or(profile, "default"), - }) + }}) } func (m *Manager) recordUsage(ctx context.Context, ev usage.Event) { diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index 8c2f19d5f..4c0230011 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -2,7 +2,6 @@ package pool import ( "encoding/json/v2" - "errors" "maps" "net/http" "net/netip" @@ -141,53 +140,6 @@ func TestJournalRotationKeepsOneBackup(t *testing.T) { } } -func TestQuotaRefusesClaimsPastCap(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 2}) - editView(m, func(v *configView) { v.maxClaims = 1 }) - - first := mustClaim(t, m, testKey) - if _, err := claimAny(t.Context(), m, testKey, ClaimOptions{TTL: time.Hour}); !errors.Is(err, ErrQuota) { - t.Fatalf("claim past cap: %v, want ErrQuota", err) - } - - if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { - t.Fatalf("Release: %v", err) - } - if _, err := claimAny(t.Context(), m, testKey, ClaimOptions{TTL: time.Hour}); err != nil { - t.Errorf("claim after release: %v", err) - } -} - -func TestTenantQuotaBindsPerTenant(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng) - setTenantCaps(t, m, map[string]int{"acme": 1, "beta": 2}) - - first := claimTenant(t, m, "acme") - if first.Tenant != "acme" { - t.Errorf("tenant %q, want acme", first.Tenant) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour, Tenant: "acme"}); !errors.Is(err, ErrQuota) { - t.Fatalf("acme past its cap: %v, want ErrQuota", err) - } - - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour, Tenant: "beta"}); err != nil { - t.Errorf("beta claim while acme is at cap: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour}); err != nil { - t.Errorf("root claim while acme is at cap: %v", err) - } - counts := m.TenantClaims() - if counts["acme"] != 1 || counts["beta"] != 1 { - t.Errorf("TenantClaims %v, want acme=1 beta=1", counts) - } - if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { - t.Fatalf("Release: %v", err) - } - claimTenant(t, m, "acme") -} - func TestTenantStampedInJournals(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -612,11 +564,7 @@ func TestEgressAuditNamesTenantAndEndsARelayedFrameCannot(t *testing.T) { Local: netip.MustParseAddrPort("198.51.100.7:40123"), Remote: netip.MustParseAddrPort("93.184.216.34:443"), }) m.Audit(t.Context(), "sb_v", []byte(`{"op":"egress","dest":"x","tenant":"other","src_addr":"198.51.100.7:1","dst_addr":"192.0.2.1:25"}`)) - raw, err := os.ReadFile(filepath.Join(dir, "audit.jsonl")) - if err != nil { - t.Fatalf("read audit journal: %v", err) - } - lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + lines := auditLines(t, dir) if len(lines) != 2 { t.Fatalf("audit lines %q, want 2", lines) } @@ -646,11 +594,7 @@ func TestARelayedFrameCannotForgeOrDropItsAuditLine(t *testing.T) { for _, frame := range frames { m.Audit(t.Context(), "sb_f", []byte(frame)) } - raw, err := os.ReadFile(filepath.Join(dir, "audit.jsonl")) - if err != nil { - t.Fatalf("read audit journal: %v", err) - } - lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + lines := auditLines(t, dir) if len(lines) != len(frames) { t.Fatalf("audit lines %q for %d frames", lines, len(frames)) } diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 4b4afc43e..83b45fdec 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -66,14 +66,13 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t if err := m.checkTemplateOwner(ctx, store.TemplateID(key.Hash()), tenant); err != nil { return types.PoolKey{}, "", err } - // A started promote must finish uncanceled ctx = context.WithoutCancel(ctx) staging, err := m.tpls.Stage(store.TemplateID(key.Hash())) if err != nil { return types.PoolKey{}, "", fmt.Errorf("stage template: %w", err) } defer func() { _ = os.RemoveAll(staging) }() - _, vmName, guestEnv, err := m.exportSource(ctx, sb, filepath.Join(staging, store.ExportDir), "") + _, vmName, guestEnv, _, err := m.exportSource(ctx, sb, filepath.Join(staging, store.ExportDir), "") if err != nil { return types.PoolKey{}, "", fmt.Errorf("promote %s: %w", sb.ID, err) } @@ -195,7 +194,7 @@ func (m *Manager) HasPoolGolden(key types.PoolKey) bool { return ok && p.goldenDir != "" } -// HasPromotedTemplate applies resolveGolden's owner test, so routing does not promise another tenant's template. +// HasPromotedTemplate reports whether tenant may clone the promoted template held under key. func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, tenant string) bool { if m.pooled(key) { return false @@ -217,7 +216,7 @@ func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, te } owner = rec.Tenant } - return owner == "" || tenantOwns(tenant, owner) + return templateVisible(tenant, owner) } func (m *Manager) loadTemplates(ctx context.Context) { @@ -375,8 +374,8 @@ func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant s if dir != "" { return goldenResolution{dir: dir, held: p, gen: gen}, nil } - if key.Net == types.NetEgress { - return goldenResolution{}, nil // never resume a live-captured template on the egress lane; cold-boot instead + if ok || key.Net == types.NetEgress { + return goldenResolution{}, nil // a pooled key waits for its golden; the egress lane never resumes a live capture } id := store.TemplateID(key.Hash()) cleanup := m.recs.rlockRec(id) @@ -393,7 +392,7 @@ func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant s cleanup() return goldenResolution{}, fmt.Errorf("decode template metadata: %w", err) } - if rec.Tenant != "" && !tenantOwns(tenant, rec.Tenant) { + if !templateVisible(tenant, rec.Tenant) { cleanup() return goldenResolution{}, nil } diff --git a/sandboxd/pool/tenants_test.go b/sandboxd/pool/tenants_test.go index 0c943ce43..cf806de97 100644 --- a/sandboxd/pool/tenants_test.go +++ b/sandboxd/pool/tenants_test.go @@ -294,11 +294,9 @@ func TestARuntimeTenantTakesItsEgressClassWithoutARestart(t *testing.T) { if status("b.test") != http.StatusOK || status("a.test") != http.StatusForbidden { t.Fatal("the live claim's door does not serve the class it was claimed under") } - next := *m.cfg - next.EgressClasses = []config.EgressClass{{Name: "desk", Egress: &egress.Policy{Allow: []egress.Rule{{Host: "a.test"}}}}} - if _, err = m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload the class: %v", err) - } + reloadWith(t, m, func(c *config.Config) { + c.EgressClasses = []config.EgressClass{{Name: "desk", Egress: &egress.Policy{Allow: []egress.Rule{{Host: "a.test"}}}}} + }) if status("a.test") != http.StatusOK { t.Error("a reload that widens the class did not reach the live claim") } @@ -391,6 +389,35 @@ func TestTwoNodesOnOneMetaStoreShareTheTenantSet(t *testing.T) { } } +func TestTenantQuotaBindsPerTenant(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng) + setTenantCaps(t, m, map[string]int{"acme": 1, "beta": 2}) + + first := claimTenant(t, m, "acme") + if first.Tenant != "acme" { + t.Errorf("tenant %q, want acme", first.Tenant) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour, Tenant: "acme"}); !errors.Is(err, ErrQuota) { + t.Fatalf("acme past its cap: %v, want ErrQuota", err) + } + + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour, Tenant: "beta"}); err != nil { + t.Errorf("beta claim while acme is at cap: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour}); err != nil { + t.Errorf("root claim while acme is at cap: %v", err) + } + counts := m.TenantClaims() + if counts["acme"] != 1 || counts["beta"] != 1 { + t.Errorf("TenantClaims %v, want acme=1 beta=1", counts) + } + if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { + t.Fatalf("Release: %v", err) + } + claimTenant(t, m, "acme") +} + func tenantOf(t *testing.T, m *Manager, token string) (string, bool) { t.Helper() r, err := m.TenantByToken(t.Context(), token) diff --git a/sandboxd/pool/tun.go b/sandboxd/pool/tun.go index 119ffa62b..f2b8dbaa1 100644 --- a/sandboxd/pool/tun.go +++ b/sandboxd/pool/tun.go @@ -140,9 +140,6 @@ func (m *Manager) releaseTun(ctx context.Context, id string, addr netip.Addr) { if err := m.tunAddrs.Release(context.WithoutCancel(ctx), id, addr); err != nil { log.WithFunc("pool.releaseTun").Warnf(ctx, "release overlay address %s of %s, retrying hourly: %v", addr, id, err) m.mu.Lock() - if m.tunReleases == nil { - m.tunReleases = map[string]netip.Addr{} - } m.tunReleases[id] = addr m.mu.Unlock() } diff --git a/sandboxd/pool/tun_test.go b/sandboxd/pool/tun_test.go index 749c7dae1..571f236ea 100644 --- a/sandboxd/pool/tun_test.go +++ b/sandboxd/pool/tun_test.go @@ -219,11 +219,7 @@ func TestAWakeIntoAPoolWithoutTunTellsTheGuestTunDown(t *testing.T) { m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) - next := *m.cfg - next.Pools = []config.PoolSpec{{PoolKey: testKey, Egress: anyPolicy}} - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { c.Pools = []config.PoolSpec{{PoolKey: testKey, Egress: anyPolicy}} }) if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("wake: %v", err) } @@ -297,9 +293,7 @@ func TestFlushClaimsPersistsTheLiveTunBytes(t *testing.T) { t.Fatal(err) } waitFor(t, func() bool { used, _ := m.out.TunUsed(sb.ID); return used > 0 }) - if err := m.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, m) if got := journaled(t, m)[sb.ID].TunBytes; got != int64(len(pkt)-tunnet.VnetHdrLen) { t.Errorf("journaled tun bytes %d, want the %d sent", got, len(pkt)-tunnet.VnetHdrLen) } diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index b3f3a9d75..336c2b65e 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -107,14 +107,10 @@ func TestRuntimeCatalogChangesAreAudited(t *testing.T) { t.Fatalf("UnregisterVolume: %v", err) } } - raw, err := os.ReadFile(filepath.Join(m.dataDir, "audit.jsonl")) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + lines := auditLines(t, m.dataDir) want := []auditFrame{{Op: "volumes", Added: []string{"data"}}, {Op: "volumes", Removed: []string{"data"}}, {Op: "volumes", Removed: []string{"data"}}} if len(lines) != len(want) { - t.Fatalf("audit records: %s", raw) + t.Fatalf("audit records: %q", lines) } for i, line := range lines { var got auditFrame diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index 9206f4260..b6b00e189 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -107,9 +107,3 @@ func newReplicaVolumeManager(t *testing.T) (*Manager, *failoverCluster) { c.volumes = []config.VolumeSpec{{Name: "data", Path: writeVolumeImage(t, "data", "original"), Writable: true}} return c.node(t, "a", t.TempDir(), keepAnHour), c } - -func disableReplication(m *Manager) { - view := *m.view.Load() - view.poolReplicate = nil - m.view.Store(&view) -} diff --git a/sandboxd/pool/watermark.go b/sandboxd/pool/watermark.go index 3aeaf0ae9..5b5815d9c 100644 --- a/sandboxd/pool/watermark.go +++ b/sandboxd/pool/watermark.go @@ -2,11 +2,13 @@ package pool import ( "context" + "errors" "math" "sync/atomic" "time" "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -22,6 +24,7 @@ const ( warmShrinkDwell = rateDecayTau ) +// pool fields are guarded by the manager mutex, except claimClones and the immutable key and hash. type pool struct { key types.PoolKey hash string @@ -57,8 +60,10 @@ type pool struct { goldenRetiredAt time.Time } -func newPool(key types.PoolKey) *pool { - return &pool{key: key, hash: key.Hash()} +func newPool(spec config.PoolSpec) *pool { + p := &pool{key: spec.PoolKey, hash: spec.Hash()} + p.applySpec(spec) + return p } func (p *pool) settled() bool { @@ -86,11 +91,11 @@ func (p *pool) noteRefillResult(now time.Time, failed bool) bool { return first } -func (p *pool) countClone(ctx context.Context, gen uint64, cloned bool, capReason string) bool { - if ctx.Err() != nil || capReason != "" || gen != p.goldenGen { +func (p *pool) countClone(ctx context.Context, gen uint64, cloneErr error, capReason string) bool { + if ctx.Err() != nil || capReason != "" || gen != p.goldenGen || errors.Is(cloneErr, engine.ErrStopping) { return false } - if cloned { + if cloneErr == nil { p.cloneFails = 0 } else { p.cloneFails++ @@ -116,7 +121,6 @@ func (p *pool) popWarm() *types.Sandbox { return sb } -// trimWarm shrinks p.warm to at most target and returns the trimmed VM names; callers hold m.mu. func (p *pool) trimWarm(target int) []string { var trim []string for len(p.warm) > target { @@ -127,7 +131,7 @@ func (p *pool) trimWarm(target int) []string { return trim } -// noteArrival counts one claim; a bin that has spanned rateBin folds its arrivals per second into the rate EWMA. Caller holds the manager mutex. +// noteArrival counts one claim; a bin that has spanned rateBin folds its arrivals per second into the rate EWMA. func (p *pool) noteArrival(now time.Time) { switch elapsed := now.Sub(p.binStart); { case p.binStart.IsZero(): @@ -141,7 +145,6 @@ func (p *pool) noteArrival(now time.Time) { p.lastArrival = now } -// noteLead folds one provision duration into the lead EWMA. Caller holds the manager mutex. func (p *pool) noteLead(d time.Duration) { if p.lead == 0 { p.lead = d @@ -150,7 +153,7 @@ func (p *pool) noteLead(d time.Duration) { p.lead = time.Duration(ewmaAlpha*float64(d) + (1-ewmaAlpha)*float64(p.lead)) } -// effectiveTarget raises the floor toward warmMax by rate × lead. Caller holds the manager mutex. +// effectiveTarget raises the floor toward warmMax by rate × lead. func (p *pool) effectiveTarget(now time.Time) int { if p.warmMax <= 0 { return p.floor @@ -169,7 +172,7 @@ func (p *pool) effectiveTarget(now time.Time) int { return max(p.floor, min(dynamic, p.warmMax)) } -// shrink trims the pool to its target once the warm count has stayed above it for warmShrinkDwell, returning the trimmed VM names. Caller holds the manager mutex. +// shrink trims the pool to its target once the warm count has stayed above it for warmShrinkDwell, returning the trimmed VM names. func (p *pool) shrink(now time.Time) []string { target := p.effectiveTarget(now) switch { diff --git a/sandboxd/poolset/pg/pg.go b/sandboxd/poolset/pg/pg.go index cb75f27fe..e55cc3afa 100644 --- a/sandboxd/poolset/pg/pg.go +++ b/sandboxd/poolset/pg/pg.go @@ -20,8 +20,12 @@ const ( channel = "sandboxd_pool_sets" ) -//go:embed schema.sql -var schema string +var ( + //go:embed schema.sql + schema string + + cas = metastore.CAS{Unavailable: poolset.ErrUnavailable} +) var _ poolset.Shared = (*Source)(nil) @@ -54,7 +58,7 @@ func (s *Source) Load(ctx context.Context) (*poolset.Set, int64, error) { case errors.Is(err, pgx.ErrNoRows): return nil, 0, nil case err != nil: - return nil, 0, fmt.Errorf("%w: %w", poolset.ErrUnavailable, err) + return nil, 0, cas.Down(err) } if err = json.Unmarshal(raw, &set.Pools); err != nil { return nil, 0, fmt.Errorf("decode cell %q pool set: %w", s.cell, err) @@ -76,7 +80,7 @@ func (s *Source) Store(ctx context.Context, set poolset.Set) (int64, error) { version = sandboxd_pool_sets.version + 1, updated_at = now() RETURNING version) SELECT version, pg_notify($4, $1) FROM up`, s.cell, set.ConfigSeed, raw, channel).Scan(&version, nil); err != nil { - return 0, fmt.Errorf("%w: %w", poolset.ErrUnavailable, err) + return 0, cas.Down(err) } return version, nil } diff --git a/sandboxd/server/exec.go b/sandboxd/server/exec.go index a48c936ae..d7bcc2f6f 100644 --- a/sandboxd/server/exec.go +++ b/sandboxd/server/exec.go @@ -7,6 +7,7 @@ import ( "io" "net" "net/http" + "slices" "strings" "time" @@ -76,10 +77,8 @@ func (s *Server) handleExec(w http.ResponseWriter, r *http.Request) { if s.mgr.AuditEnabled() { s.mgr.Audit(ctx, id, frame) } - frame = append(frame, '\n') stdinClose, _ := wire.EncodeRequest(wire.StdinClose{}) - frame = append(frame, stdinClose...) - frame = append(frame, '\n') + frame = slices.Concat(frame, []byte{'\n'}, stdinClose, []byte{'\n'}) if _, err := guest.Write(frame); err != nil { writeErr(w, http.StatusBadGateway, "guest agent unreachable") return diff --git a/sandboxd/server/exec_test.go b/sandboxd/server/exec_test.go index 8965ca275..4581004e6 100644 --- a/sandboxd/server/exec_test.go +++ b/sandboxd/server/exec_test.go @@ -10,6 +10,7 @@ import ( "net" "net/http" "net/http/httptest" + "strconv" "strings" "sync/atomic" "testing" @@ -18,6 +19,8 @@ import ( "github.com/cocoonstack/sandbox/protocol/wire" ) +const startedFrame = `{"type":"started","pid":7}` + "\n" + func TestExecReturnsBufferedOutput(t *testing.T) { var gotExec, gotStdinClose wire.Request ts, _ := newRelayServer(t, func(conn net.Conn) { @@ -33,10 +36,10 @@ func TestExecReturnsBufferedOutput(t *testing.T) { return } gotStdinClose, _ = wire.DecodeRequest(bytes.TrimSpace(frame)) - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) _, _ = io.WriteString(conn, stdoutFrame("v22\n")) _, _ = io.WriteString(conn, `{"type":"stderr","data":"`+base64.StdEncoding.EncodeToString([]byte("warn\n"))+`"}`+"\n") - _, _ = io.WriteString(conn, `{"type":"exit","code":3}`+"\n") + _, _ = io.WriteString(conn, exitFrame(3)) }) status, body := postExec(t, ts, `{"argv":["node","-v"],"cwd":"/work","env":{"A":"1"},"timeout_seconds":5}`) if status != http.StatusOK { @@ -64,9 +67,9 @@ func TestExecKeepsInvalidUTF8OutputAsReplacementCharacters(t *testing.T) { r := bufio.NewReader(conn) _, _ = r.ReadBytes('\n') _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) _, _ = io.WriteString(conn, stdoutFrame("ok\xff\n")) - _, _ = io.WriteString(conn, `{"type":"exit","code":0}`+"\n") + _, _ = io.WriteString(conn, exitFrame(0)) }) status, body := postExec(t, ts, `{"argv":["cat","blob"],"timeout_seconds":5}`) if status != http.StatusOK { @@ -82,16 +85,7 @@ func TestExecKeepsInvalidUTF8OutputAsReplacementCharacters(t *testing.T) { } func TestExecAuditsTheFrameWithoutItsDelimiter(t *testing.T) { - ts, srv := newRelayServer(t, func(conn net.Conn) { - defer conn.Close() - r := bufio.NewReader(conn) - for range 2 { - if _, err := r.ReadBytes('\n'); err != nil { - return - } - } - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n"+`{"type":"exit","code":0}`+"\n") - }) + ts, srv := newRelayServer(t, okGuest) audited := make(chan []byte, 1) srv.mgr.(*fakeManager).audited = func(_ string, frame []byte) { audited <- bytes.Clone(frame) } if status, body := postExec(t, ts, `{"argv":["true"]}`); status != http.StatusOK { @@ -104,16 +98,7 @@ func TestExecAuditsTheFrameWithoutItsDelimiter(t *testing.T) { } func TestExecRunsUnderAnOverflowingTimeout(t *testing.T) { - ts, _ := newRelayServer(t, func(conn net.Conn) { - defer conn.Close() - r := bufio.NewReader(conn) - for range 2 { - if _, err := r.ReadBytes('\n'); err != nil { - return - } - } - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n"+`{"type":"exit","code":0}`+"\n") - }) + ts, _ := newRelayServer(t, okGuest) status, body := postExec(t, ts, `{"argv":["true"],"timeout_seconds":10000000000}`) if status != http.StatusOK { t.Fatalf("status %d, want 200: a huge timeout_seconds must not wrap into an expired deadline: %s", status, body) @@ -147,32 +132,28 @@ func TestExecTimeoutKillsTheCommand(t *testing.T) { ts, _ := newRelayServer(t, killWatchGuest(killed, func(conn net.Conn, r *bufio.Reader) { defer close(closed) _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) _, _ = r.ReadBytes('\n') _, _ = r.ReadBytes('\n') })) if status, _ := postExec(t, ts, `{"argv":["sleep","60"],"timeout_seconds":1}`); status != http.StatusGatewayTimeout { t.Fatalf("status %d, want 504", status) } - select { - case <-closed: - case <-time.After(3 * time.Second): - t.Fatal("guest conn still open after the timeout") - } + await(t, closed, "guest conn still open after the timeout") expectKill(t, killed, "timeout") } -func TestExecRejectsUnknownFields(t *testing.T) { - ts, _ := newRelayServer(t, func(conn net.Conn) { _ = conn.Close() }) - if status, _ := postExec(t, ts, `{"argv":["whoami"],"user":"nobody"}`); status != http.StatusBadRequest { - t.Errorf("status %d, want 400", status) - } -} - -func TestExecRejectsEmptyArgv(t *testing.T) { - ts, _ := newRelayServer(t, func(conn net.Conn) { _ = conn.Close() }) - if status, _ := postExec(t, ts, `{"argv":[]}`); status != http.StatusBadRequest { - t.Errorf("status %d, want 400", status) +func TestExecRejectsBadRequests(t *testing.T) { + for _, tt := range []struct{ name, body string }{ + {"unknown fields", `{"argv":["whoami"],"user":"nobody"}`}, + {"empty argv", `{"argv":[]}`}, + } { + t.Run(tt.name, func(t *testing.T) { + ts, _ := newRelayServer(t, func(conn net.Conn) { _ = conn.Close() }) + if status, _ := postExec(t, ts, tt.body); status != http.StatusBadRequest { + t.Errorf("status %d, want 400", status) + } + }) } } @@ -200,7 +181,7 @@ func TestExecGuestClosingAfterTheStartWakesOnce(t *testing.T) { r := bufio.NewReader(guestEnd) _, _ = r.ReadBytes('\n') _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(guestEnd, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(guestEnd, startedFrame) })) if status, body := postExec(t, ts, `{"argv":["sleep","60"]}`); status != http.StatusBadGateway { @@ -226,7 +207,7 @@ func TestExecTimeoutStartsAfterWake(t *testing.T) { r := bufio.NewReader(guestEnd) _, _ = r.ReadBytes('\n') _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(guestEnd, `{"type":"exit","code":0}`+"\n") + _, _ = io.WriteString(guestEnd, exitFrame(0)) })) if status, body := postExec(t, ts, `{"argv":["true"],"timeout_seconds":1}`); status != http.StatusOK { @@ -241,7 +222,7 @@ func TestExecOverTheNodeBudgetIs503AndKillsTheCommand(t *testing.T) { killed := make(chan uint32, 1) ts, srv := newRelayServer(t, killWatchGuest(killed, func(conn net.Conn, r *bufio.Reader) { _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) _, _ = io.WriteString(conn, stdoutFrame("x")) _, _ = io.Copy(io.Discard, r) })) @@ -261,7 +242,7 @@ func TestExecOverTheOutputCapIs413AndKillsTheCommand(t *testing.T) { chunk := stdoutFrame(strings.Repeat("x", 1<<20)) ts, _ := newRelayServer(t, killWatchGuest(killed, func(conn net.Conn, r *bufio.Reader) { _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) for range execOutputCap>>20 + 1 { if _, err := io.WriteString(conn, chunk); err != nil { return @@ -280,9 +261,9 @@ func TestExecReturnsItsBudgetAfterAnswering(t *testing.T) { defer conn.Close() r := bufio.NewReader(conn) _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) _, _ = io.WriteString(conn, stdoutFrame("v22\n")) - _, _ = io.WriteString(conn, `{"type":"exit","code":0}`+"\n") + _, _ = io.WriteString(conn, exitFrame(0)) }) if status, body := postExec(t, ts, `{"argv":["node","-v"]}`); status != http.StatusOK { t.Fatalf("status %d, want 200: %s", status, body) @@ -328,14 +309,24 @@ func killWatchGuest(killed chan<- uint32, first func(net.Conn, *bufio.Reader)) g func expectKill(t *testing.T, killed <-chan uint32, after string) { t.Helper() - select { - case pid := <-killed: - if pid != 7 { - t.Errorf("killed pid %d, want 7", pid) + if pid := await(t, killed, "no kill reached the guest after the "+after); pid != 7 { + t.Errorf("killed pid %d, want 7", pid) + } +} + +func okGuest(conn net.Conn) { + defer conn.Close() + r := bufio.NewReader(conn) + for range 2 { + if _, err := r.ReadBytes('\n'); err != nil { + return } - case <-time.After(3 * time.Second): - t.Fatalf("no kill reached the guest after the %s", after) } + _, _ = io.WriteString(conn, startedFrame+exitFrame(0)) +} + +func exitFrame(code int) string { + return `{"type":"exit","code":` + strconv.Itoa(code) + `}` + "\n" } func stdoutFrame(data string) string { diff --git a/sandboxd/server/port_test.go b/sandboxd/server/port_test.go index 440dc2384..694d262cd 100644 --- a/sandboxd/server/port_test.go +++ b/sandboxd/server/port_test.go @@ -148,37 +148,40 @@ func TestPortRelayEndsOnGuestClose(t *testing.T) { } func TestSpliceTearsDownWhenTheClientDies(t *testing.T) { - srv := New("", "node:7777", &fakeManager{}, &fakeDialer{}, nil, nil, nil, nil) - t.Cleanup(srv.CloseRelays) - client, clientPeer := net.Pipe() - t.Cleanup(func() { _ = client.Close(); _ = clientPeer.Close() }) - guest, guestPeer := net.Pipe() - t.Cleanup(func() { _ = guestPeer.Close() }) - go func() { _, _ = io.Copy(io.Discard, clientPeer) }() + synctest.Test(t, func(t *testing.T) { + srv := New("", "node:7777", &fakeManager{}, &fakeDialer{}, nil, nil, nil, nil) + t.Cleanup(srv.CloseRelays) + client, clientPeer := net.Pipe() + t.Cleanup(func() { _ = client.Close(); _ = clientPeer.Close() }) + guest, guestPeer := net.Pipe() + t.Cleanup(func() { _ = guestPeer.Close() }) + go func() { _, _ = io.Copy(io.Discard, clientPeer) }() - guestClosed := make(chan error, 1) - go func() { - _, err := guestPeer.Read(make([]byte, 1)) - guestClosed <- err - }() + guestClosed := make(chan error, 1) + go func() { + _, err := guestPeer.Read(make([]byte, 1)) + guestClosed <- err + }() - ended := make(chan struct{}) - go func() { - defer close(ended) - srv.splice(client, iotest.ErrReader(errors.New("client vanished")), guest, - switchingProtocolsTCP, tunnelEnds) - }() + ended := make(chan struct{}) + go func() { + defer close(ended) + srv.splice(client, iotest.ErrReader(errors.New("client vanished")), guest, + switchingProtocolsTCP, tunnelEnds) + }() - select { - case err := <-guestClosed: - if err == nil { - t.Error("guest read succeeded; the relay should have closed it") + synctest.Wait() + select { + case err := <-guestClosed: + if err == nil { + t.Error("guest read succeeded; the relay should have closed it") + } + default: + t.Fatal("a dead client left the guest connection open; the relay holds the sandbox") } - case <-time.After(3 * time.Second): - t.Fatal("a dead client left the guest connection open; the relay holds the sandbox") - } - _ = client.Close() - <-ended + _ = client.Close() + <-ended + }) } func TestSpliceTearsDownWhenTheGuestFails(t *testing.T) { diff --git a/sandboxd/server/preview.go b/sandboxd/server/preview.go index 683dda87b..8e97a2cba 100644 --- a/sandboxd/server/preview.go +++ b/sandboxd/server/preview.go @@ -47,7 +47,10 @@ type PreviewServer struct { // NewPreviewServer serves preview URLs signed with secret and minted under base. func NewPreviewServer(secret, base, owner string, mgr PreviewManager) *PreviewServer { - p := &PreviewServer{secret: []byte(secret), base: base, owner: owner, mgr: mgr} + if !strings.Contains(base, "://") { + base = "http://" + base + } + p := &PreviewServer{secret: []byte(secret), base: strings.TrimRight(base, "/"), owner: owner, mgr: mgr} p.transport = &http.Transport{ // A pooled guest conn skips PreviewDial, and with it wakeResolved and the Transition lock DisableKeepAlives: true, @@ -72,11 +75,7 @@ func (p *PreviewServer) Mint(id string, port uint16, ttl time.Duration) string { payload, _ := json.Marshal(claims) enc := base64.RawURLEncoding.EncodeToString(payload) token := enc + "." + p.sign(enc) - base := p.base - if !strings.Contains(base, "://") { - base = "http://" + base - } - return fmt.Sprintf("%s/p/%s/", strings.TrimRight(base, "/"), token) + return fmt.Sprintf("%s/p/%s/", p.base, token) } // Handler serves preview requests and health checks on preview_listen. diff --git a/sandboxd/server/preview_test.go b/sandboxd/server/preview_test.go index 06d349c6b..68e2a93b4 100644 --- a/sandboxd/server/preview_test.go +++ b/sandboxd/server/preview_test.go @@ -79,46 +79,22 @@ func TestPreviewProxiesToGuest(t *testing.T) { } } -func TestPreviewRevokedWhenDialFails(t *testing.T) { - ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ - dial: func(string, uint16) (net.Conn, error) { return nil, net.ErrClosed }, - }) - ts := httptest.NewServer(ps.Handler()) - t.Cleanup(ts.Close) - token := mintToken(ps, "sb_gone", 3000, time.Hour) - - resp, err := http.Get(ts.URL + "/p/" + token + "/") - if err != nil { - t.Fatalf("get: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusBadGateway { - t.Errorf("status %d, want 502 for a gone sandbox", resp.StatusCode) - } -} - -func TestPreviewAnswers404ForAReleasedSandbox(t *testing.T) { - ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ - dial: func(string, uint16) (net.Conn, error) { return nil, pool.ErrUnknownSandbox }, - }) - ts := httptest.NewServer(ps.Handler()) - t.Cleanup(ts.Close) - - resp, err := http.Get(ts.URL + "/p/" + mintToken(ps, "sb_released", 3000, time.Hour) + "/") - if err != nil { - t.Fatalf("get: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusNotFound { - t.Errorf("status %d, want 404 for a released sandbox", resp.StatusCode) - } -} - -func TestPreviewAnswers503WhileAClaimCannotServe(t *testing.T) { - for _, dialErr := range []error{pool.ErrLeaseLost, pool.ErrEvacuating} { - t.Run(dialErr.Error(), func(t *testing.T) { +func TestPreviewAnswersTheStatusOfAFailedDial(t *testing.T) { + for _, tt := range []struct { + name string + dialErr error + status int + wantRetry bool + }{ + {"gone sandbox", net.ErrClosed, http.StatusBadGateway, false}, + {"released sandbox", pool.ErrUnknownSandbox, http.StatusNotFound, false}, + {pool.ErrLeaseLost.Error(), pool.ErrLeaseLost, http.StatusServiceUnavailable, true}, + {pool.ErrEvacuating.Error(), pool.ErrEvacuating, http.StatusServiceUnavailable, true}, + {"quota", fmt.Errorf("wake sb_1: %w", pool.ErrQuota), http.StatusTooManyRequests, false}, + } { + t.Run(tt.name, func(t *testing.T) { ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ - dial: func(string, uint16) (net.Conn, error) { return nil, dialErr }, + dial: func(string, uint16) (net.Conn, error) { return nil, tt.dialErr }, }) ts := httptest.NewServer(ps.Handler()) t.Cleanup(ts.Close) @@ -129,33 +105,16 @@ func TestPreviewAnswers503WhileAClaimCannotServe(t *testing.T) { t.Fatalf("get: %v", err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") == "" { - t.Errorf("status %d retry-after %q, want 503 with a retry hint", resp.StatusCode, resp.Header.Get("Retry-After")) + if resp.StatusCode != tt.status { + t.Errorf("status %d, want %d", resp.StatusCode, tt.status) + } + if tt.wantRetry && resp.Header.Get("Retry-After") == "" { + t.Error("no Retry-After on a 503, want a retry hint") } }) } } -func TestPreviewAnswers429AtACap(t *testing.T) { - ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ - dial: func(string, uint16) (net.Conn, error) { - return nil, fmt.Errorf("wake sb_1: %w", pool.ErrQuota) - }, - }) - ts := httptest.NewServer(ps.Handler()) - t.Cleanup(ts.Close) - token := mintToken(ps, "sb_1", 3000, time.Hour) - - resp, err := http.Get(ts.URL + "/p/" + token + "/") - if err != nil { - t.Fatalf("get: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusTooManyRequests { - t.Errorf("status %d, want 429", resp.StatusCode) - } -} - func TestPreviewRechecksClaimForEveryRequest(t *testing.T) { guestAddr := newGuestServer(t, func(*http.Request) string { return "guest" }) diff --git a/sandboxd/server/reader.go b/sandboxd/server/reader.go index c6abe8caa..12dd40424 100644 --- a/sandboxd/server/reader.go +++ b/sandboxd/server/reader.go @@ -11,7 +11,6 @@ import ( "net/http" "slices" "sync/atomic" - "time" "github.com/projecteru2/core/log" @@ -195,16 +194,11 @@ func (s *Server) serveSleeping(ctx context.Context, w http.ResponseWriter, id, t if !ok { return } - release, ok := s.trackRelay(client, io.NopCloser(nil)) + release, ok := s.openRelay(client, io.NopCloser(nil), switchingProtocols) if !ok { return } defer release() - _ = client.SetDeadline(time.Time{}) - keepAlive(client) - if _, err := client.Write(switchingProtocols); err != nil { - return - } rt := &router{s: s, ctx: ctx, id: id, token: token, client: client, br: bufio.NewReaderSize(clientReader(client, clientBuf), routeBuf)} guest, done, ok := rt.run() if !ok { diff --git a/sandboxd/server/reader_test.go b/sandboxd/server/reader_test.go index ec18eb22c..5244d0201 100644 --- a/sandboxd/server/reader_test.go +++ b/sandboxd/server/reader_test.go @@ -97,11 +97,7 @@ func TestSleepingLaterWriteMovesTheConnectionToTheGuest(t *testing.T) { send(t, conn, execFrame) expectLine(t, r, "guest") - select { - case <-env.readerClosed: - case <-time.After(3 * time.Second): - t.Fatal("the reader connection stayed open after the claim woke") - } + await(t, env.readerClosed, "the reader connection stayed open after the claim woke") send(t, conn, statFrame) expectLine(t, r, "guest") if wakes, dials := env.reader.counts(); wakes != 1 || dials != 1 { @@ -122,11 +118,7 @@ func TestSleepingReaderStopsServingOnceTheClaimWoke(t *testing.T) { env.reader.setSource("") send(t, conn, statFrame) expectLine(t, r, "guest") - select { - case <-env.readerClosed: - case <-time.After(3 * time.Second): - t.Fatal("the stale reader connection stayed open") - } + await(t, env.readerClosed, "the stale reader connection stayed open") } func TestSleepingConnectionMovesToAFreshReaderWhenTheClaimSleepsAgain(t *testing.T) { @@ -138,11 +130,7 @@ func TestSleepingConnectionMovesToAFreshReaderWhenTheClaimSleepsAgain(t *testing env.reader.setSource("snap-2") send(t, conn, readFrame) expectLine(t, r, "reader") - select { - case <-env.readerClosed: - case <-time.After(3 * time.Second): - t.Fatal("the stale reader connection stayed open") - } + await(t, env.readerClosed, "the stale reader connection stayed open") if wakes, dials := env.reader.counts(); wakes != 0 || dials != 2 { t.Errorf("wakes=%d reader dials=%d, want a fresh reader for the new sleep and no wake", wakes, dials) } @@ -311,11 +299,7 @@ func TestSleepingReaderClosesWhenItsClientLeavesMidReply(t *testing.T) { send(t, conn, readFrame) send(t, conn, statFrame) _ = conn.Close() - select { - case <-env.readerClosed: - case <-time.After(5 * time.Second): - t.Fatal("the reader connection outlived its client") - } + await(t, env.readerClosed, "the reader connection outlived its client") } func TestSleepingCloseRelaysEndsReaderConnections(t *testing.T) { @@ -329,11 +313,7 @@ func TestSleepingCloseRelaysEndsReaderConnections(t *testing.T) { env.srv.CloseRelays() close(done) }() - select { - case <-done: - case <-time.After(3 * time.Second): - t.Fatal("CloseRelays hung on a reader connection") - } + await(t, done, "CloseRelays hung on a reader connection") if _, err := r.ReadString('\n'); !errors.Is(err, io.EOF) { t.Errorf("client after CloseRelays: %v, want EOF", err) } diff --git a/sandboxd/server/relay.go b/sandboxd/server/relay.go index 7193e3118..e752bf146 100644 --- a/sandboxd/server/relay.go +++ b/sandboxd/server/relay.go @@ -109,23 +109,16 @@ func (s *Server) audited(ctx context.Context, id string, clientR io.Reader) io.R } func (s *Server) splice(client net.Conn, clientR io.Reader, guest net.Conn, hello []byte, ends relayEnds) { - release, ok := s.trackRelay(client, guest) + release, ok := s.openRelay(client, guest, hello) if !ok { return } defer release() - - // The http server may have armed a header-read deadline on this conn - _ = client.SetDeadline(time.Time{}) - keepAlive(client) - if _, err := client.Write(hello); err != nil { - return - } copyBoth(client, clientR, guest, ends, s.noSplice) } -// trackRelay registers a hijacked pair for CloseRelays; !ok means the server is already draining. -func (s *Server) trackRelay(client net.Conn, guest io.Closer) (func(), bool) { +// openRelay registers a hijacked pair for CloseRelays and sends hello; !ok means the server is draining or the client is gone. +func (s *Server) openRelay(client net.Conn, guest io.Closer, hello []byte) (func(), bool) { s.relayMu.Lock() if s.relayClosed { s.relayMu.Unlock() @@ -136,14 +129,22 @@ func (s *Server) trackRelay(client net.Conn, guest io.Closer) (func(), bool) { s.relays[client] = guest s.relayWG.Add(1) s.relayMu.Unlock() - return func() { + release := func() { _ = client.Close() _ = guest.Close() s.relayMu.Lock() delete(s.relays, client) s.relayMu.Unlock() s.relayWG.Done() - }, true + } + // The http server may have armed a header-read deadline on this conn + _ = client.SetDeadline(time.Time{}) + keepAlive(client) + if _, err := client.Write(hello); err != nil { + release() + return nil, false + } + return release, true } func (s *Server) retargetRelay(client net.Conn, peer io.Closer) { diff --git a/sandboxd/server/relay_test.go b/sandboxd/server/relay_test.go index 3ff0ca4b9..5d66735de 100644 --- a/sandboxd/server/relay_test.go +++ b/sandboxd/server/relay_test.go @@ -125,11 +125,7 @@ func TestRelayClientDisconnectClosesGuest(t *testing.T) { conn, _ := upgradeConn(t, ts) _ = conn.Close() - select { - case <-guestClosed: - case <-time.After(3 * time.Second): - t.Fatal("guest side not closed after client disconnect") - } + await(t, guestClosed, "guest side not closed after client disconnect") } func TestRelayEndsWhenItsClaimIsFenced(t *testing.T) { diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index a5b1740c4..fa7473325 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -918,8 +918,7 @@ func (s *Server) requireToken(next http.HandlerFunc) http.HandlerFunc { tenant, err := s.resolveScope(r) switch { case errors.Is(err, pool.ErrTenantStoreDown): - w.Header().Set("Retry-After", retryAfterSeconds) - writeErr(w, http.StatusServiceUnavailable, "tenant store down; retry") + writePoolErr(w, err) return case err != nil: writeErr(w, http.StatusUnauthorized, "invalid api token") diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index c3664982b..238b09eb3 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -495,15 +495,17 @@ func TestSandboxesNarrowToAClaimRef(t *testing.T) { {"/v1/sandboxes?claim_ref=team-a%2Fdemo", "team-a/demo"}, {"/v1/sandboxes", ""}, } { - got := "unset" - mgr := &fakeManager{sandboxIndex: func(_, claimRef string) []pool.SandboxSummary { - got = claimRef - return nil - }} - ts := newTenantTestServer(t, "root", nil, mgr, nil) - if code := statusOf(t, http.MethodGet, ts.URL+tt.path, "root", ""); code != http.StatusOK || got != tt.want { - t.Errorf("%s: status %d, manager saw claim_ref %q, want %q", tt.path, code, got, tt.want) - } + t.Run(tt.path, func(t *testing.T) { + got := "unset" + mgr := &fakeManager{sandboxIndex: func(_, claimRef string) []pool.SandboxSummary { + got = claimRef + return nil + }} + ts := newTenantTestServer(t, "root", nil, mgr, nil) + if code := statusOf(t, http.MethodGet, ts.URL+tt.path, "root", ""); code != http.StatusOK || got != tt.want { + t.Errorf("%s: status %d, manager saw claim_ref %q, want %q", tt.path, code, got, tt.want) + } + }) } } @@ -689,9 +691,11 @@ func TestInstanceMetadataVerb(t *testing.T) { {"over 4 KiB", "sekret", "sb_1", `{"a":"` + strings.Repeat("x", 4<<10) + `"}`, http.StatusBadRequest}, {"unknown sandbox", "sekret", "sb_2", valid, http.StatusNotFound}, } { - if code := put(tt.token, tt.id, tt.body); code != tt.want { - t.Errorf("%s: status %d, want %d", tt.name, code, tt.want) - } + t.Run(tt.name, func(t *testing.T) { + if code := put(tt.token, tt.id, tt.body); code != tt.want { + t.Errorf("%s: status %d, want %d", tt.name, code, tt.want) + } + }) } for _, err := range []error{pool.ErrPaused, pool.ErrArchived, pool.ErrNoInstanceMetadata} { fail = err @@ -711,12 +715,14 @@ func TestDrainEndpoints(t *testing.T) { {http.MethodPost, true}, {http.MethodDelete, false}, } { - resp := doReq(t, tt.method, ts.URL+"/v1/drain", "sekret", "") - info := decodeJSON[InfoResponse](t, resp.Body) - resp.Body.Close() - if resp.StatusCode != http.StatusOK || info.Draining != tt.draining || mgr.draining != tt.draining { - t.Fatalf("%s: status=%d draining=%v mgr=%v, want 200/%v", tt.method, resp.StatusCode, info.Draining, mgr.draining, tt.draining) - } + t.Run(tt.method, func(t *testing.T) { + resp := doReq(t, tt.method, ts.URL+"/v1/drain", "sekret", "") + info := decodeJSON[InfoResponse](t, resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || info.Draining != tt.draining || mgr.draining != tt.draining { + t.Fatalf("%s: status=%d draining=%v mgr=%v, want 200/%v", tt.method, resp.StatusCode, info.Draining, mgr.draining, tt.draining) + } + }) } } @@ -731,15 +737,17 @@ func TestCordonEndpoints(t *testing.T) { {http.MethodPost, true, "sandboxd_cordoned 1\n"}, {http.MethodDelete, false, "sandboxd_cordoned 0\n"}, } { - resp := doReq(t, tt.method, ts.URL+"/v1/cordon", "sekret", "") - info := decodeJSON[InfoResponse](t, resp.Body) - resp.Body.Close() - if resp.StatusCode != http.StatusOK || info.Cordoned != tt.cordoned || mgr.cordoned != tt.cordoned || info.Draining { - t.Fatalf("%s: status=%d info=%+v mgr=%v, want 200/%v and no drain", tt.method, resp.StatusCode, info, mgr.cordoned, tt.cordoned) - } - if _, body := bodyOf(t, http.MethodGet, ts.URL+"/metrics", "sekret", ""); !strings.Contains(string(body), tt.metric) { - t.Errorf("%s: metrics miss %q", tt.method, tt.metric) - } + t.Run(tt.method, func(t *testing.T) { + resp := doReq(t, tt.method, ts.URL+"/v1/cordon", "sekret", "") + info := decodeJSON[InfoResponse](t, resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || info.Cordoned != tt.cordoned || mgr.cordoned != tt.cordoned || info.Draining { + t.Fatalf("%s: status=%d info=%+v mgr=%v, want 200/%v and no drain", tt.method, resp.StatusCode, info, mgr.cordoned, tt.cordoned) + } + if _, body := bodyOf(t, http.MethodGet, ts.URL+"/metrics", "sekret", ""); !strings.Contains(string(body), tt.metric) { + t.Errorf("%s: metrics miss %q", tt.method, tt.metric) + } + }) } } @@ -751,11 +759,13 @@ func TestCordonPersistFailureAnswers500AndKeepsState(t *testing.T) { {http.MethodPost, false}, {http.MethodDelete, true}, } { - mgr := &fakeManager{cordoned: tt.cordoned, cordonErr: errors.New("disk gone")} - ts := newTestServer(t, "sekret", mgr, nil) - if code := statusOf(t, tt.method, ts.URL+"/v1/cordon", "sekret", ""); code != http.StatusInternalServerError || mgr.cordoned != tt.cordoned { - t.Errorf("%s: status %d cordoned=%v, want 500 and cordoned=%v", tt.method, code, mgr.cordoned, tt.cordoned) - } + t.Run(tt.method, func(t *testing.T) { + mgr := &fakeManager{cordoned: tt.cordoned, cordonErr: errors.New("disk gone")} + ts := newTestServer(t, "sekret", mgr, nil) + if code := statusOf(t, tt.method, ts.URL+"/v1/cordon", "sekret", ""); code != http.StatusInternalServerError || mgr.cordoned != tt.cordoned { + t.Errorf("%s: status %d cordoned=%v, want 500 and cordoned=%v", tt.method, code, mgr.cordoned, tt.cordoned) + } + }) } } @@ -2105,17 +2115,19 @@ func TestClaimRejectsMetadataOrClaimRefOverABound(t *testing.T) { {"/v1/claim", `{"template":"rt:24.04","volumes":[{"name":"data"}],"claim_ref":"` + strings.Repeat("r", 513) + `"}`, "claim_ref must"}, {"/v1/sandboxes/sb_1/fork", `{"token":"tok","count":1,"claim_ref_prefix":"ns/\u0007"}`, "claim_ref_prefix must"}, } { - mgr := &fakeManager{} - ts := newTestServer(t, "sekret", mgr, nil) - resp := postJSON(t, ts.URL+tt.path, "sekret", tt.body) - got := decodeJSON[errorReply](t, resp.Body) - resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest || !strings.Contains(got.Error, tt.want) { - t.Errorf("%s: status %d error %q, want 400 naming %q", tt.path, resp.StatusCode, got.Error, tt.want) - } - if mgr.warmCalls+mgr.provisionCalls+mgr.healCalls != 0 || mgr.gotMetadata != nil { - t.Errorf("%s: the manager ran on a rejected claim", tt.path) - } + t.Run(tt.want+" "+tt.path, func(t *testing.T) { + mgr := &fakeManager{} + ts := newTestServer(t, "sekret", mgr, nil) + resp := postJSON(t, ts.URL+tt.path, "sekret", tt.body) + got := decodeJSON[errorReply](t, resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest || !strings.Contains(got.Error, tt.want) { + t.Errorf("%s: status %d error %q, want 400 naming %q", tt.path, resp.StatusCode, got.Error, tt.want) + } + if mgr.warmCalls+mgr.provisionCalls+mgr.healCalls != 0 || mgr.gotMetadata != nil { + t.Errorf("%s: the manager ran on a rejected claim", tt.path) + } + }) } } @@ -2140,12 +2152,14 @@ func TestClaimThreadsMetadataAndEnv(t *testing.T) { {"checkpoint", "/v1/checkpoints/ck_00000000000000aa/claim", `{"metadata":{"team":"a"},"env":{"GW":{"value":"Bearer k","guest":false}}}`, func(f *fakeManager) { f.claimCheckpoint = ckpt }}, {"checkpoint heal", "/v1/checkpoints/ck_00000000000000aa/claim", `{"metadata":{"team":"a"},"env":{"GW":{"value":"Bearer k","guest":false}}}`, func(f *fakeManager) { f.healCheckpoint = ckpt }}, } { - mgr := &fakeManager{} - tt.setup(mgr) - ts := newTestServer(t, "sekret", mgr, nil) - if code := statusOf(t, http.MethodPost, ts.URL+tt.path, "sekret", tt.body); code != http.StatusOK || !maps.Equal(mgr.gotMetadata, want) || !mgr.gotEnv.Equal(wantEnv) { - t.Errorf("%s: status %d, manager saw metadata %v env %v, want 200 and %v %v", tt.name, code, mgr.gotMetadata, mgr.gotEnv, want, wantEnv) - } + t.Run(tt.name, func(t *testing.T) { + mgr := &fakeManager{} + tt.setup(mgr) + ts := newTestServer(t, "sekret", mgr, nil) + if code := statusOf(t, http.MethodPost, ts.URL+tt.path, "sekret", tt.body); code != http.StatusOK || !maps.Equal(mgr.gotMetadata, want) || !mgr.gotEnv.Equal(wantEnv) { + t.Errorf("%s: status %d, manager saw metadata %v env %v, want 200 and %v %v", tt.name, code, mgr.gotMetadata, mgr.gotEnv, want, wantEnv) + } + }) } } @@ -2164,11 +2178,13 @@ func TestSandboxesFilterByMetadata(t *testing.T) { {"?metadata=%3Dv", http.StatusBadRequest, nil}, {"?metadata=a%3D1&metadata=a%3D2", http.StatusBadRequest, nil}, } { - mgr := &fakeManager{} - ts := newTestServer(t, "root", mgr, nil) - if code := statusOf(t, http.MethodGet, ts.URL+"/v1/sandboxes"+tt.query, "root", ""); code != tt.code || !maps.Equal(mgr.gotMetadata, tt.want) { - t.Errorf("%q: status %d, manager saw %v, want %d and %v", tt.query, code, mgr.gotMetadata, tt.code, tt.want) - } + t.Run(tt.query, func(t *testing.T) { + mgr := &fakeManager{} + ts := newTestServer(t, "root", mgr, nil) + if code := statusOf(t, http.MethodGet, ts.URL+"/v1/sandboxes"+tt.query, "root", ""); code != tt.code || !maps.Equal(mgr.gotMetadata, tt.want) { + t.Errorf("%q: status %d, manager saw %v, want %d and %v", tt.query, code, mgr.gotMetadata, tt.code, tt.want) + } + }) } } @@ -2191,22 +2207,24 @@ func TestExpireActionThreadsThroughEveryDecoder(t *testing.T) { f.renew = func(string, string, time.Duration) (time.Time, error) { return time.Now(), nil } }}, } { - mgr := &fakeManager{} - tt.setup(mgr) - ts := newTestServer(t, "sekret", mgr, nil) - if code := statusOf(t, http.MethodPost, ts.URL+tt.path, "sekret", tt.body); code != http.StatusOK || mgr.gotOnExpire != types.ExpireArchive { - t.Errorf("%s: status %d, manager saw on_expire %q, want 200 and archive", tt.name, code, mgr.gotOnExpire) - } + t.Run(tt.name, func(t *testing.T) { + mgr := &fakeManager{} + tt.setup(mgr) + ts := newTestServer(t, "sekret", mgr, nil) + if code := statusOf(t, http.MethodPost, ts.URL+tt.path, "sekret", tt.body); code != http.StatusOK || mgr.gotOnExpire != types.ExpireArchive { + t.Errorf("%s: status %d, manager saw on_expire %q, want 200 and archive", tt.name, code, mgr.gotOnExpire) + } - mgr = &fakeManager{} - tt.setup(mgr) - ts = newTestServer(t, "sekret", mgr, nil) - resp := postJSON(t, ts.URL+tt.path, "sekret", strings.Replace(tt.body, `"archive"`, `"pause"`, 1)) - got := decodeJSON[errorReply](t, resp.Body) - resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest || !strings.Contains(got.Error, "on_expire") { - t.Errorf("%s with on_expire pause: status %d error %q, want 400 naming on_expire", tt.name, resp.StatusCode, got.Error) - } + mgr = &fakeManager{} + tt.setup(mgr) + ts = newTestServer(t, "sekret", mgr, nil) + resp := postJSON(t, ts.URL+tt.path, "sekret", strings.Replace(tt.body, `"archive"`, `"pause"`, 1)) + got := decodeJSON[errorReply](t, resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest || !strings.Contains(got.Error, "on_expire") { + t.Errorf("%s with on_expire pause: status %d error %q, want 400 naming on_expire", tt.name, resp.StatusCode, got.Error) + } + }) } } @@ -2244,9 +2262,11 @@ func TestTunVerbIsRootOnlyAndPassesOnlyTheNamedFields(t *testing.T) { {"unknown field", "sekret", "sb_1", `{"quota": 1}`, http.StatusBadRequest}, {"unknown sandbox", "sekret", "sb_2", `{"enabled": true}`, http.StatusNotFound}, } { - if code := put(tt.token, tt.id, tt.body); code != tt.want { - t.Errorf("%s: status %d, want %d", tt.name, code, tt.want) - } + t.Run(tt.name, func(t *testing.T) { + if code := put(tt.token, tt.id, tt.body); code != tt.want { + t.Errorf("%s: status %d, want %d", tt.name, code, tt.want) + } + }) } } @@ -2266,6 +2286,8 @@ type checkpointClaimFunc func(string) (*types.Sandbox, error) type idActionFunc func(string) error +type guestFunc func(net.Conn) + type fakeManager struct { counters pool.Counters registerVolume func(context.Context, string, types.VolumeRegistration) error @@ -2769,8 +2791,6 @@ func (f *fakeManager) record(o pool.ClaimOptions) { f.gotNoEgress = o.NoEgress } -type guestFunc func(net.Conn) - type fakeDialer struct { dial func(ctx context.Context, sock string) (net.Conn, error) } @@ -2933,6 +2953,18 @@ func postJSON(t *testing.T, url, token, body string) *http.Response { return doReq(t, http.MethodPost, url, token, body) } +func await[T any](t *testing.T, ch <-chan T, msg string) T { + t.Helper() + select { + case v := <-ch: + return v + case <-time.After(5 * time.Second): + t.Fatal(msg) + var zero T + return zero + } +} + func scrapeMetrics(t *testing.T, base string) string { t.Helper() _, raw := bodyOf(t, http.MethodGet, base+"/metrics", "", "") diff --git a/sandboxd/store/dir/dir_test.go b/sandboxd/store/dir/dir_test.go index 80200f02e..42bf7eb5c 100644 --- a/sandboxd/store/dir/dir_test.go +++ b/sandboxd/store/dir/dir_test.go @@ -46,12 +46,8 @@ func TestFetchPinsGenerationAcrossStoreInstances(t *testing.T) { t.Fatalf("fetch digest %q, want first generation %q", digest, firstDigest) } secondDigest := mustPublishDigested(t, writer, testID, "second") - if _, statErr := os.Stat(filepath.Join(dir, "disk.img")); statErr != nil { - t.Fatalf("first generation disturbed by re-publish: %v", statErr) - } - if _, statErr := os.Stat(firstDigestPath); statErr != nil { - t.Fatalf("first generation digest disturbed by re-publish: %v", statErr) - } + assertPresentf(t, filepath.Join(dir, "disk.img"), "first generation disturbed by re-publish") + assertPresentf(t, firstDigestPath, "first generation digest disturbed by re-publish") _, meta, digest = mustFetch(t, reader, testID) if string(meta) != storetest.MetaJSON("second") { @@ -77,9 +73,7 @@ func TestPlainPublishReplacesDigestWithEmpty(t *testing.T) { if firstDigest == "" { t.Fatal("digested generation returned an empty digest") } - if _, statErr := os.Stat(firstDigestPath); statErr != nil { - t.Fatalf("superseded digest removed inside the grace: %v", statErr) - } + assertPresentf(t, firstDigestPath, "superseded digest removed inside the grace") mustPublish(t, st, testID, "digested") if _, _, digest = mustFetch(t, st, testID); digest != "" { t.Errorf("a plain publish of the digested meta kept digest %q, want empty", digest) @@ -270,14 +264,10 @@ func TestSweepGenerationsPairsCurrentAndSupersededDigests(t *testing.T) { t.Fatalf("SweepGenerations: %v", err) } for _, path := range []string{firstGen, firstSidecar} { - if _, statErr := os.Stat(path); !errors.Is(statErr, fs.ErrNotExist) { - t.Errorf("superseded entry %s survived sweep: %v", filepath.Base(path), statErr) - } + assertGonef(t, path, "superseded entry %s survived sweep", filepath.Base(path)) } for _, path := range []string{secondGen, secondSidecar, labels} { - if _, statErr := os.Stat(path); statErr != nil { - t.Errorf("current entry %s was swept: %v", filepath.Base(path), statErr) - } + assertPresentf(t, path, "current entry %s was swept", filepath.Base(path)) } _, meta, digest := mustFetch(t, st, testID) if string(meta) != storetest.MetaJSON("second") || digest != secondDigest { @@ -304,17 +294,13 @@ func TestPublishRetriesAfterExpiredInstall(t *testing.T) { if sweepErr := st.SweepGenerations(t.Context()); sweepErr != nil { t.Fatalf("sweep expired generation: %v", sweepErr) } - if _, statErr := os.Stat(gen); !errors.Is(statErr, fs.ErrNotExist) { - t.Fatalf("expired generation survived sweep: %v", statErr) - } + assertGonef(t, gen, "expired generation survived sweep") mustPublish(t, st, testID, "same") backdate(t, gen) mustPublish(t, st, testID, "same") dir, _, _ := mustFetch(t, st, testID) - if _, err := os.Stat(filepath.Join(dir, "disk.img")); err != nil { - t.Fatalf("export after retried publish: %v", err) - } + assertPresentf(t, filepath.Join(dir, "disk.img"), "export after retried publish") } func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { @@ -325,26 +311,18 @@ func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { mustPublish(t, st, testID, "first") mustPublish(t, st, testID, "second") mustPublish(t, st, testID, "third") - if _, statErr := os.Stat(gen1); statErr != nil { - t.Fatalf("superseded generation reclaimed inside the grace: %v", statErr) - } + assertPresentf(t, gen1, "superseded generation reclaimed inside the grace") setAge(t, gen1, store.GenerationGrace+time.Minute) setAge(t, gen2, store.GenerationGrace/2) mustPublish(t, st, testID, "fourth") - if _, statErr := os.Stat(gen1); !errors.Is(statErr, fs.ErrNotExist) { - t.Fatalf("generation past its supersession grace survived publish: %v", statErr) - } - if _, statErr := os.Stat(gen2); statErr != nil { - t.Fatalf("generation inside its supersession grace was reclaimed: %v", statErr) - } + assertGonef(t, gen1, "generation past its supersession grace survived publish") + assertPresentf(t, gen2, "generation inside its supersession grace was reclaimed") dir, meta, _ := mustFetch(t, st, testID) if string(meta) != storetest.MetaJSON("fourth") { t.Fatalf("fetch meta %q, want the current generation", meta) } - if _, err := os.Stat(filepath.Join(dir, "disk.img")); err != nil { - t.Fatalf("current export after sweep: %v", err) - } + assertPresentf(t, filepath.Join(dir, "disk.img"), "current export after sweep") } func TestSweepSparesPublishingGeneration(t *testing.T) { @@ -358,9 +336,7 @@ func TestSweepSparesPublishingGeneration(t *testing.T) { if sweepErr := sweeper.SweepStaging(); sweepErr != nil { t.Fatalf("sweep during publish: %v", sweepErr) } - if _, statErr := os.Stat(gen); statErr != nil { - t.Fatalf("publishing generation was swept: %v", statErr) - } + assertPresentf(t, gen, "publishing generation was swept") tmp := filepath.Join(root, testID, store.MetaFile+".tmp") if writeErr := os.WriteFile(tmp, meta, 0o600); writeErr != nil { @@ -413,15 +389,9 @@ func TestSweepReclaimsOnlyAgedUncommittedGenerations(t *testing.T) { if sweepErr := sweeper.SweepStaging(); sweepErr != nil { t.Fatalf("sweep old: %v", sweepErr) } - if _, statErr := os.Stat(orphan); !errors.Is(statErr, fs.ErrNotExist) { - t.Fatalf("aged uncommitted generation survived: %v", statErr) - } - if _, statErr := os.Stat(filepath.Join(root, orphanID)); statErr != nil { - t.Fatalf("orphan record shell was removed: %v", statErr) - } - if _, statErr := os.Stat(fresh); statErr != nil { - t.Fatalf("publishing generation was swept: %v", statErr) - } + assertGonef(t, orphan, "aged uncommitted generation survived") + assertPresentf(t, filepath.Join(root, orphanID), "orphan record shell was removed") + assertPresentf(t, fresh, "publishing generation was swept") if removeErr := os.RemoveAll(commitBlocker); removeErr != nil { t.Fatalf("remove commit blocker: %v", removeErr) @@ -448,17 +418,13 @@ func TestSweepStagingSparesAPeersFreshStaging(t *testing.T) { if sweepErr := restarted.SweepStaging(); sweepErr != nil { t.Fatalf("sweep: %v", sweepErr) } - if _, statErr := os.Stat(filepath.Join(staging, store.MetaFile)); statErr != nil { - t.Fatalf("a peer's in-progress staging was swept: %v", statErr) - } + assertPresentf(t, filepath.Join(staging, store.MetaFile), "a peer's in-progress staging was swept") backdate(t, staging) if sweepErr := restarted.SweepStaging(); sweepErr != nil { t.Fatalf("sweep aged: %v", sweepErr) } - if _, statErr := os.Stat(staging); !errors.Is(statErr, fs.ErrNotExist) { - t.Errorf("aged staging survived: %v", statErr) - } + assertGonef(t, staging, "aged staging survived") } func TestSweepGenerationsRemovesOnlyAgedStaging(t *testing.T) { @@ -474,12 +440,8 @@ func TestSweepGenerationsRemovesOnlyAgedStaging(t *testing.T) { t.Fatalf("SweepGenerations: %v", err) } - if _, err := os.Stat(aged); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("staging aged past the grace survived the periodic sweep: %v", err) - } - if _, err := os.Stat(fresh); err != nil { - t.Errorf("fresh staging was swept: %v", err) - } + assertGonef(t, aged, "staging aged past the grace survived the periodic sweep") + assertPresentf(t, fresh, "fresh staging was swept") } func TestSweepGenerationsToleratesConcurrentDelete(t *testing.T) { @@ -713,3 +675,17 @@ func setAge(t *testing.T, path string, age time.Duration) { t.Fatalf("backdate %s: %v", path, err) } } + +func assertGonef(t *testing.T, path, format string, args ...any) { + t.Helper() + if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) { + t.Errorf(format+": %v", append(args, err)...) + } +} + +func assertPresentf(t *testing.T, path, format string, args ...any) { + t.Helper() + if _, err := os.Stat(path); err != nil { + t.Errorf(format+": %v", append(args, err)...) + } +} diff --git a/sandboxd/store/peer/broadcast_test.go b/sandboxd/store/peer/broadcast_test.go index 8f5d0fa32..88d0e89d1 100644 --- a/sandboxd/store/peer/broadcast_test.go +++ b/sandboxd/store/peer/broadcast_test.go @@ -28,32 +28,27 @@ func TestBroadcastDeleteSendsNoForward(t *testing.T) { } func TestBroadcastDeleteFansOutToEveryPeer(t *testing.T) { - var hits atomic.Int32 - newSrv := func() *httptest.Server { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusNoContent) - })) - t.Cleanup(srv.Close) - return srv + var addrs []string + var hits []*atomic.Int32 + for range 3 { + srv, n := countingServer(t, http.StatusNoContent) + addrs, hits = append(addrs, srv.URL), append(hits, n) } - addrs := []string{newSrv().URL, newSrv().URL, newSrv().URL} b := &Broadcaster{Peers: func() []string { return addrs }} b.Delete(t.Context(), testID) - if got := hits.Load(); got != 3 { + got := 0 + for _, n := range hits { + got += int(n.Load()) + } + if got != 3 { t.Errorf("peers hit %d times, want 3 (one per distinct peer)", got) } } func TestBroadcastDeleteDedupsRepeatedAddrs(t *testing.T) { - var hits atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusNoContent) - })) - defer srv.Close() + srv, hits := countingServer(t, http.StatusNoContent) b := &Broadcaster{Peers: func() []string { return []string{srv.URL, srv.URL, srv.URL} }} b.Delete(t.Context(), testID) @@ -68,12 +63,7 @@ func TestBroadcastDeleteSwallowsFailures(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) })) defer fail.Close() - var reached atomic.Int32 - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - reached.Add(1) - w.WriteHeader(http.StatusNoContent) - })) - defer ok.Close() + ok, reached := countingServer(t, http.StatusNoContent) b := &Broadcaster{Peers: func() []string { return []string{fail.URL, ok.URL, "127.0.0.1:1"} }} b.Delete(t.Context(), testID) @@ -81,8 +71,3 @@ func TestBroadcastDeleteSwallowsFailures(t *testing.T) { t.Errorf("healthy peer hit %d times, want 1 (fan-out continues past a failure)", got) } } - -func TestBroadcastDeleteNoPeersNoOp(t *testing.T) { - b := &Broadcaster{Peers: func() []string { return nil }} - b.Delete(t.Context(), testID) -} diff --git a/sandboxd/store/peer/move_test.go b/sandboxd/store/peer/move_test.go index aa069829d..26b6d335a 100644 --- a/sandboxd/store/peer/move_test.go +++ b/sandboxd/store/peer/move_test.go @@ -12,7 +12,6 @@ import ( "path/filepath" "slices" "strings" - "syscall" "testing" "time" @@ -62,7 +61,7 @@ func TestSendRoundShipsTheListedFilesInOrderAndKeepsSparseFilesSparse(t *testing t.Fatalf("ReceiveRound: %v", err) } for _, name := range names { - if sha256OfFile(t, filepath.Join(dst, name)) != sha256OfFile(t, filepath.Join(src, name)) { + if storetest.FileSum(t, filepath.Join(dst, name)) != storetest.FileSum(t, filepath.Join(src, name)) { t.Errorf("%s differs from the source", name) } } @@ -70,7 +69,7 @@ func TestSendRoundShipsTheListedFilesInOrderAndKeepsSparseFilesSparse(t *testing if err != nil || info.Size() != size { t.Fatalf("received cow.raw = %v, %v, want %d bytes", info, err, size) } - if allocated := info.Sys().(*syscall.Stat_t).Blocks * 512; allocated > sent { + if allocated := storetest.AllocatedBytes(t, filepath.Join(dst, "cow.raw")); allocated > sent { t.Errorf("received cow.raw allocates %d bytes, more than the %d sent", allocated, sent) } entries, _ := os.ReadDir(dst) @@ -99,7 +98,7 @@ func TestReceiveRoundReplacesFilesOfTheSameName(t *testing.T) { if body, err := os.ReadFile(filepath.Join(dst, "state.json")); err != nil || string(body) != "new" { t.Errorf("state.json = %q, %v, want the new round's bytes", body, err) } - if sha256OfFile(t, filepath.Join(dst, "cow.raw")) != sha256OfFile(t, filepath.Join(src, "cow.raw")) { + if storetest.FileSum(t, filepath.Join(dst, "cow.raw")) != storetest.FileSum(t, filepath.Join(src, "cow.raw")) { t.Error("cow.raw keeps bytes of the replaced file in its holes") } } diff --git a/sandboxd/store/peer/probe.go b/sandboxd/store/peer/probe.go index 1aa94c260..d30a03914 100644 --- a/sandboxd/store/peer/probe.go +++ b/sandboxd/store/peer/probe.go @@ -17,6 +17,9 @@ import ( ) const ( + // ProbeHeader carries the base64 probe MAC when a ProbeKey is configured. + ProbeHeader = "X-Cocoon-Probe" + probeTimeout = time.Second fanOutTimeout = 2 * time.Second maxRedirectOwners = 3 @@ -26,9 +29,6 @@ const ( // probeGrace bounds how long a fan-out waits for more owners once it has its first. probeGrace = 150 * time.Millisecond - // ProbeHeader carries the base64 probe MAC when a ProbeKey is configured. - ProbeHeader = "X-Cocoon-Probe" - // probeBucketSeconds is the coarse time step the probe MAC is bound to. probeBucketSeconds = 30 ) diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go index 588e8ac76..687da6556 100644 --- a/sandboxd/store/peer/probe_test.go +++ b/sandboxd/store/peer/probe_test.go @@ -3,10 +3,13 @@ package peer import ( "net/http" "net/http/httptest" + "strings" "sync" "sync/atomic" "testing" "time" + + "github.com/cocoonstack/sandbox/sandboxd/store/storetest" ) func TestOwnersReturnsOnly200(t *testing.T) { @@ -29,21 +32,6 @@ func TestOwnersAllMissIsEmpty(t *testing.T) { } } -func TestOwnersHungPeerExcludedWithoutBlockingOthers(t *testing.T) { - hung := hungServer(t) - ok := statusServer(t, http.StatusOK) - - start := time.Now() - p := &HTTPProber{Peers: func() []string { return []string{hung.URL, ok.URL} }} - owners := p.Owners(t.Context(), testID) - if elapsed := time.Since(start); elapsed > 2*time.Second { - t.Errorf("Owners took %v, want bounded by the per-probe timeout", elapsed) - } - if len(owners) != 1 || owners[0] != ok.URL { - t.Errorf("owners = %v, want [%s]", owners, ok.URL) - } -} - func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { hung := hungServer(t) ok := statusServer(t, http.StatusOK) @@ -60,12 +48,7 @@ func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { } func TestOwnersDedupsAddrs(t *testing.T) { - var hits atomic.Int32 - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() + ok, hits := countingServer(t, http.StatusOK) p := &HTTPProber{Peers: func() []string { return []string{ok.URL, ok.URL, ok.URL} }} owners := p.Owners(t.Context(), testID) @@ -77,15 +60,15 @@ func TestOwnersDedupsAddrs(t *testing.T) { } } -func TestOwnersCapsAtThree(t *testing.T) { +func TestOwnersCapsAtTheRedirectLimit(t *testing.T) { var addrs []string - for range 4 { + for range maxRedirectOwners + 1 { addrs = append(addrs, statusServer(t, http.StatusOK).URL) } p := &HTTPProber{Peers: func() []string { return addrs }} - if owners := p.Owners(t.Context(), testID); len(owners) != 3 { - t.Errorf("owners = %v, want capped at 3", owners) + if owners := p.Owners(t.Context(), testID); len(owners) != maxRedirectOwners { + t.Errorf("owners = %v, want capped at %d", owners, maxRedirectOwners) } } @@ -136,18 +119,23 @@ func TestHealOwnersIsExhaustiveDespiteAFastOwner(t *testing.T) { func TestOwnersCoalescesConcurrentProbes(t *testing.T) { var hits atomic.Int32 + hold := make(chan struct{}) + release := sync.OnceFunc(func() { close(hold) }) ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { hits.Add(1) - time.Sleep(50 * time.Millisecond) + <-hold w.WriteHeader(http.StatusOK) })) defer ok.Close() + defer release() p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }} var wg sync.WaitGroup for range 10 { wg.Go(func() { p.Owners(t.Context(), testID) }) } + storetest.WaitStacks(t, 10, func(stack string) bool { return strings.Contains(stack, "singleflight.(*Group).Do(") }) + release() wg.Wait() if got := hits.Load(); got != 1 { t.Errorf("peer hit %d times by 10 concurrent callers, want 1 (coalesced)", got) @@ -182,16 +170,10 @@ func TestOwnersCacheCollapsesRepeatedRedirects(t *testing.T) { } func TestOwnersCacheExpires(t *testing.T) { - var hits atomic.Int32 - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() + ok, hits := countingServer(t, http.StatusOK) - p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }, cacheTTL: 10 * time.Millisecond} + p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }, cacheTTL: -time.Second} p.Owners(t.Context(), testID) - time.Sleep(20 * time.Millisecond) p.Owners(t.Context(), testID) if got := hits.Load(); got != 2 { t.Errorf("peer hit %d times across two Owners calls spanning the TTL, want 2 (expired)", got) @@ -199,12 +181,7 @@ func TestOwnersCacheExpires(t *testing.T) { } func TestForgetEvictsCachedEntry(t *testing.T) { - var hits atomic.Int32 - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() + ok, hits := countingServer(t, http.StatusOK) p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }} p.Owners(t.Context(), testID) @@ -216,12 +193,7 @@ func TestForgetEvictsCachedEntry(t *testing.T) { } func TestForgetOtherIDLeavesCacheAlone(t *testing.T) { - var hits atomic.Int32 - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() + ok, hits := countingServer(t, http.StatusOK) p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }} p.Owners(t.Context(), testID) @@ -281,6 +253,17 @@ func statusServer(t *testing.T, code int) *httptest.Server { return srv } +func countingServer(t *testing.T, code int) (*httptest.Server, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(code) + })) + t.Cleanup(srv.Close) + return srv, &hits +} + func hungServer(t *testing.T) *httptest.Server { t.Helper() block := make(chan struct{}) diff --git a/sandboxd/store/peer/transport_test.go b/sandboxd/store/peer/transport_test.go index 45f1ffbc2..18fbe752b 100644 --- a/sandboxd/store/peer/transport_test.go +++ b/sandboxd/store/peer/transport_test.go @@ -3,7 +3,6 @@ package peer import ( "archive/tar" "bytes" - "crypto/sha256" "errors" "io" "net/http" @@ -12,7 +11,6 @@ import ( "path/filepath" "slices" "strings" - "syscall" "testing" "time" @@ -236,12 +234,9 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { } meta := []byte(`{"id":"ck_0000000000000001","key":{"template":"t"}}`) - var buf bytes.Buffer - if err := TarRecord(exportDir, meta, &buf); err != nil { - t.Fatalf("TarRecord: %v", err) - } + buf := tarOf(t, exportDir, meta) dst := t.TempDir() - if err := Untar(&buf, dst); err != nil { + if err := Untar(buf, dst); err != nil { t.Fatalf("Untar: %v", err) } @@ -269,6 +264,9 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { if _, err := os.Stat(filepath.Join(dst, "memory-ranges")); err == nil { t.Error("export contents landed at the record root; they must be under export/") } + if _, err := os.Stat(filepath.Join(dst, recordTrailer)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("completion marker was written to disk (%v); it must be consumed, not stored", err) + } } func TestUntarRejectsRecordWithoutCompletionMarker(t *testing.T) { @@ -286,27 +284,6 @@ func TestUntarRejectsRecordWithoutCompletionMarker(t *testing.T) { } } -func TestTarRecordUntarRoundTripComplete(t *testing.T) { - exportDir := t.TempDir() - if err := os.WriteFile(filepath.Join(exportDir, "mem"), []byte("state"), 0o600); err != nil { - t.Fatalf("setup: %v", err) - } - var buf bytes.Buffer - if err := TarRecord(exportDir, []byte(`{"id":"ck_1"}`), &buf); err != nil { - t.Fatalf("TarRecord: %v", err) - } - dst := t.TempDir() - if err := Untar(&buf, dst); err != nil { - t.Fatalf("Untar: %v", err) - } - if _, err := os.Stat(filepath.Join(dst, recordTrailer)); !errors.Is(err, os.ErrNotExist) { - t.Errorf("completion marker was written to disk (%v); it must be consumed, not stored", err) - } - if _, err := os.Stat(filepath.Join(dst, "export", "mem")); err != nil { - t.Errorf("export content missing after round trip: %v", err) - } -} - func TestTarRecordKeepsSparseFilesSparse(t *testing.T) { size := 10*store.DigestChunkSize + 4096 exportDir := t.TempDir() @@ -318,18 +295,15 @@ func TestTarRecordKeepsSparseFilesSparse(t *testing.T) { if err := os.WriteFile(filepath.Join(exportDir, "snapshot.json"), []byte("{}"), 0o600); err != nil { t.Fatalf("write snapshot: %v", err) } - want := sha256OfFile(t, src) + want := storetest.FileSum(t, src) - var buf bytes.Buffer - if err := TarRecord(exportDir, []byte(`{"id":"ck_0000000000000001"}`), &buf); err != nil { - t.Fatalf("TarRecord: %v", err) - } + buf := tarOf(t, exportDir, []byte(`{"id":"ck_0000000000000001"}`)) sent := int64(buf.Len()) if limit := 3*store.DigestChunkSize + 1<<20; sent > limit { t.Errorf("stream carries %d bytes for a %d-byte file with 3 data chunks, want at most %d", sent, size, limit) } dst := t.TempDir() - if err := Untar(&buf, dst); err != nil { + if err := Untar(buf, dst); err != nil { t.Fatalf("Untar: %v", err) } got := filepath.Join(dst, "export", "cow.raw") @@ -337,10 +311,10 @@ func TestTarRecordKeepsSparseFilesSparse(t *testing.T) { if err != nil || info.Size() != size { t.Fatalf("pulled size = %v, %v, want %d", info, err, size) } - if sha256OfFile(t, got) != want { + if storetest.FileSum(t, got) != want { t.Error("pulled content differs from the source") } - if allocated := info.Sys().(*syscall.Stat_t).Blocks * 512; allocated > sent { + if allocated := storetest.AllocatedBytes(t, got); allocated > sent { t.Errorf("pulled file allocates %d bytes, more than the %d sent", allocated, sent) } if body, err := os.ReadFile(filepath.Join(dst, "export", "snapshot.json")); err != nil || string(body) != "{}" { @@ -353,11 +327,7 @@ func TestUntarRejectsDamagedChunk(t *testing.T) { if err := os.WriteFile(filepath.Join(exportDir, "mem"), []byte("guest-pages-payload"), 0o600); err != nil { t.Fatalf("setup: %v", err) } - var buf bytes.Buffer - if err := TarRecord(exportDir, []byte(`{"id":"ck_1"}`), &buf); err != nil { - t.Fatalf("TarRecord: %v", err) - } - raw := buf.Bytes() + raw := tarOf(t, exportDir, []byte(`{"id":"ck_1"}`)).Bytes() raw[bytes.Index(raw, []byte("guest-pages-payload"))] ^= 0xff err := Untar(bytes.NewReader(raw), t.TempDir()) if err == nil || !strings.Contains(err.Error(), "checksum") { @@ -377,16 +347,11 @@ func tarDir(src string, w io.Writer) error { return tw.Close() } -func sha256OfFile(t *testing.T, path string) [sha256.Size]byte { +func tarOf(t *testing.T, exportDir string, meta []byte) *bytes.Buffer { t.Helper() - f, err := os.Open(path) - if err != nil { - t.Fatalf("open %s: %v", path, err) - } - defer func() { _ = f.Close() }() - h := sha256.New() - if _, err = io.Copy(h, f); err != nil { - t.Fatalf("hash %s: %v", path, err) + var buf bytes.Buffer + if err := TarRecord(exportDir, meta, &buf); err != nil { + t.Fatalf("TarRecord: %v", err) } - return [sha256.Size]byte(h.Sum(nil)) + return &buf } diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index e7f3b341e..57cdf01bd 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -19,7 +19,6 @@ import ( "strconv" "strings" "sync" - "syscall" "testing" "time" @@ -27,6 +26,8 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/store/storetest" ) +const _ = uint(store.GenerationGrace - transferBudget - 1) + func TestS3BackendContract(t *testing.T) { _, st := newFakeStore(t) storetest.RunContract(t, st) @@ -139,7 +140,6 @@ func TestFetchReportsMetaWithoutExportAsNotFound(t *testing.T) { func TestFetchOutlivesTheCallerThatStartedIt(t *testing.T) { const id = "ck_00000000000000dd" - metaKey := "ck/" + id + "/" + store.MetaFile fake, st, release := newGatedStore(t, "") publishRecord(t, st, id, metaJSON(id), "payload") fake.resetRequests() @@ -156,10 +156,9 @@ func TestFetchOutlivesTheCallerThatStartedIt(t *testing.T) { _, _, _, err := st.Fetch(t.Context(), id) joined <- err }() - for fake.requestCount(http.MethodGet, metaKey) < 2 { - time.Sleep(time.Millisecond) - } - time.Sleep(50 * time.Millisecond) + storetest.WaitStacks(t, 2, func(stack string) bool { + return strings.Contains(stack, "s3.(*Store).Fetch(") && !strings.Contains(stack, "readMeta") + }) hangUp() select { case err := <-started: @@ -437,7 +436,7 @@ func TestSparseExportStaysSparseThroughS3(t *testing.T) { size := 12*store.DigestChunkSize + 4096 meta := metaJSON(id) staging, src := stageSparseRecord(t, st, id, meta, size, map[int64][]byte{0: []byte("head"), 3*store.DigestChunkSize + 100: []byte("mid"), 11*store.DigestChunkSize + 10: []byte("tail")}) - srcSum := fileSum(t, src) + srcSum := storetest.FileSum(t, src) if err := st.Publish(t.Context(), staging, id, ""); err != nil { t.Fatalf("Publish: %v", err) } @@ -455,10 +454,10 @@ func TestSparseExportStaysSparseThroughS3(t *testing.T) { if err != nil || info.Size() != size { t.Fatalf("fetched size = %v, %v, want %d", info, err, size) } - if fileSum(t, got) != srcSum { + if storetest.FileSum(t, got) != srcSum { t.Error("fetched content differs from the source") } - if blocks := allocatedBytes(t, got); blocks > uploaded+(1<<20) { + if blocks := storetest.AllocatedBytes(t, got); blocks > uploaded+(1<<20) { t.Errorf("fetched file allocates %d bytes, want at most the %d uploaded", blocks, uploaded) } } @@ -570,7 +569,7 @@ func TestLineageDedupUploadsOnlyChangedChunks(t *testing.T) { t.Helper() before, _ := fake.chunkPuts("ck/chunks/") staging, src := stageSparseRecord(t, st, id, fmt.Appendf(nil, `{"id":"%s","gen":%d}`, id, gen), 8*cs, writes) - srcSum = fileSum(t, src) + srcSum = storetest.FileSum(t, src) if err := st.Publish(t.Context(), staging, id, scope); err != nil { t.Fatalf("Publish %s: %v", id, err) } @@ -596,7 +595,7 @@ func TestLineageDedupUploadsOnlyChangedChunks(t *testing.T) { if err != nil { t.Fatalf("Fetch: %v", err) } - if fileSum(t, filepath.Join(dir, "disk.img")) != srcSum { + if storetest.FileSum(t, filepath.Join(dir, "disk.img")) != srcSum { t.Error("fetched content differs from the source") } } @@ -802,13 +801,6 @@ func TestDeleteRemovesTheLineageMarker(t *testing.T) { } } -func TestTransferBudgetStaysBelowTheReuseWindow(t *testing.T) { - budget := transferBudget - if budget >= store.GenerationGrace { - t.Errorf("transfer budget %v must stay below the generation grace %v", budget, store.GenerationGrace) - } -} - func TestFetchFileDownloadsOnlyTheRequestedFile(t *testing.T) { const id = "ck_00000000000000aa" meta := metaJSON(id) @@ -871,7 +863,7 @@ func TestFetchFileKeepsHoles(t *testing.T) { _, st := newFakeStore(t) size := 12*store.DigestChunkSize + 4096 staging, src := stageSparseRecord(t, st, id, metaJSON(id), size, map[int64][]byte{0: []byte("head"), 11*store.DigestChunkSize + 10: []byte("tail")}) - srcSum := fileSum(t, src) + srcSum := storetest.FileSum(t, src) if err := st.Publish(t.Context(), staging, id, ""); err != nil { t.Fatalf("Publish: %v", err) } @@ -883,10 +875,10 @@ func TestFetchFileKeepsHoles(t *testing.T) { if info, err := os.Stat(dst); err != nil || info.Size() != size { t.Fatalf("fetched size = %v, %v, want %d", info, err, size) } - if fileSum(t, dst) != srcSum { + if storetest.FileSum(t, dst) != srcSum { t.Error("fetched content differs from the source") } - if blocks, limit := allocatedBytes(t, dst), 2*store.DigestChunkSize+1<<20; blocks > limit { + if blocks, limit := storetest.AllocatedBytes(t, dst), 2*store.DigestChunkSize+1<<20; blocks > limit { t.Errorf("fetched file allocates %d bytes, want at most %d for two data chunks", blocks, limit) } } @@ -926,7 +918,7 @@ func TestRaisedConcurrencyReachesItsLimitAndKeepsItsConnections(t *testing.T) { writes[i*store.DigestChunkSize] = []byte{byte(i + 1)} } staging, src := stageSparseRecord(t, st, id, metaJSON(id), chunks*store.DigestChunkSize, writes) - srcSum := fileSum(t, src) + srcSum := storetest.FileSum(t, src) gate.arm(http.MethodPut, chunks) if err := st.Publish(t.Context(), staging, id, ""); err != nil { @@ -941,7 +933,7 @@ func TestRaisedConcurrencyReachesItsLimitAndKeepsItsConnections(t *testing.T) { if err := st.FetchFile(t.Context(), id, "disk.img", dst); err != nil { t.Fatalf("FetchFile round %d: %v", round, err) } - if fileSum(t, dst) != srcSum { + if storetest.FileSum(t, dst) != srcSum { t.Fatalf("FetchFile round %d: content differs from the source", round) } peak, opened := gate.result() @@ -1246,29 +1238,6 @@ func stageSparseRecord(t *testing.T, st *Store, id string, meta []byte, size int return staging, storetest.WriteSparse(t, filepath.Join(staging, store.ExportDir, "disk.img"), size, writes) } -func fileSum(t *testing.T, path string) [sha256.Size]byte { - t.Helper() - f, err := os.Open(path) - if err != nil { - t.Fatalf("open %s: %v", path, err) - } - defer func() { _ = f.Close() }() - h := sha256.New() - if _, err = io.Copy(h, f); err != nil { - t.Fatalf("hash %s: %v", path, err) - } - return [sha256.Size]byte(h.Sum(nil)) -} - -func allocatedBytes(t *testing.T, path string) int64 { - t.Helper() - info, err := os.Stat(path) - if err != nil { - t.Fatalf("stat %s: %v", path, err) - } - return info.Sys().(*syscall.Stat_t).Blocks * 512 -} - func denseChunkSums(t *testing.T, path string) [][sha256.Size]byte { t.Helper() f, err := os.Open(path) diff --git a/sandboxd/store/storetest/storetest.go b/sandboxd/store/storetest/storetest.go index 7e700f52c..9bf6781ec 100644 --- a/sandboxd/store/storetest/storetest.go +++ b/sandboxd/store/storetest/storetest.go @@ -3,11 +3,17 @@ package storetest import ( "context" + "crypto/sha256" "errors" + "io" "os" "path/filepath" + "runtime" "slices" + "strings" + "syscall" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/store" ) @@ -147,6 +153,47 @@ func WriteSparse(t testing.TB, path string, size int64, writes map[int64][]byte) return path } +func FileSum(t testing.TB, path string) [sha256.Size]byte { + t.Helper() + f, err := os.Open(path) //nolint:gosec // test path + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err = io.Copy(h, f); err != nil { + t.Fatalf("hash %s: %v", path, err) + } + return [sha256.Size]byte(h.Sum(nil)) +} + +func AllocatedBytes(t testing.TB, path string) int64 { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + return info.Sys().(*syscall.Stat_t).Blocks * 512 +} + +func WaitStacks(t testing.TB, want int, match func(stack string) bool) { + t.Helper() + buf := make([]byte, 1<<20) + for range 5000 { + matched := 0 + for stack := range strings.SplitSeq(string(buf[:runtime.Stack(buf, true)]), "\n\n") { + if match(stack) { + matched++ + } + } + if matched >= want { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("fewer than %d goroutines reached the expected state", want) +} + func runFetchFileContract(t *testing.T, st store.Store, id string) { t.Helper() ctx := t.Context() diff --git a/sandboxd/tenants/pg/pg.go b/sandboxd/tenants/pg/pg.go index ba21160fe..18a5bb44d 100644 --- a/sandboxd/tenants/pg/pg.go +++ b/sandboxd/tenants/pg/pg.go @@ -36,8 +36,12 @@ const ( uniqueViolation = "23505" ) -//go:embed schema.sql -var schema string +var ( + //go:embed schema.sql + schema string + + cas = metastore.CAS{Unavailable: tenants.ErrUnavailable} +) type entry struct { rec *tenants.TenantRecord @@ -104,13 +108,13 @@ func (s *Source) Resolve(ctx context.Context, sum [sha256.Size]byte) (*tenants.T case !s.cold.Allow(): return nil, fmt.Errorf("%w: cold lookups over %d/s", tenants.ErrUnavailable, coldLookups) } - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - r, err := scanRecord(s.pool.QueryRow(qctx, `SELECT name, token_sha256, max_claims, egress_class FROM sandboxd_tenants WHERE token_sha256 = $1`, sum[:])) + r, ok, err := cas.One(ctx, s.pool, scanRecord, `SELECT name, token_sha256, max_claims, egress_class FROM sandboxd_tenants WHERE token_sha256 = $1`, sum[:]) s.mu.Lock() defer s.mu.Unlock() switch { - case errors.Is(err, pgx.ErrNoRows): + case err != nil: + return nil, err + case !ok: if s.seq == start { if len(s.misses) >= maxCached { clear(s.misses) @@ -118,8 +122,6 @@ func (s *Source) Resolve(ctx context.Context, sum [sha256.Size]byte) (*tenants.T s.misses[sum] = time.Now().Add(negativeTTL) } return nil, tenants.ErrUnknown - case err != nil: - return nil, unavailable(err) } if s.fresh(r.Name, start) { s.cache(&r, sum) @@ -161,17 +163,10 @@ func (s *Source) Lookup(ctx context.Context, name string) (tenants.TenantRecord, } func (s *Source) List(ctx context.Context, after string, limit int) ([]tenants.TenantRecord, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - rows, err := s.pool.Query(qctx, `SELECT name, max_claims, egress_class FROM sandboxd_tenants WHERE name > $1 ORDER BY name LIMIT $2`, after, limit) - if err != nil { - return nil, unavailable(err) - } - out, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (tenants.TenantRecord, error) { + return cas.All(ctx, s.pool, func(row pgx.Row) (tenants.TenantRecord, error) { var r tenants.TenantRecord return r, row.Scan(&r.Name, &r.MaxClaims, &r.EgressClass) - }) - return out, unavailable(err) + }, `SELECT name, max_claims, egress_class FROM sandboxd_tenants WHERE name > $1 ORDER BY name LIMIT $2`, after, limit) } func (s *Source) Put(ctx context.Context, r tenants.TenantRecord) (tenants.Change, error) { @@ -258,13 +253,13 @@ func (s *Source) Classes(ctx context.Context) (map[string]int, error) { defer cancel() rows, err := s.pool.Query(qctx, `SELECT egress_class, count(*) FROM sandboxd_tenants WHERE egress_class <> '' GROUP BY egress_class`) if err != nil { - return nil, unavailable(err) + return nil, cas.Down(err) } out := map[string]int{} var class string var n int if _, err = pgx.ForEachRow(rows, []any{&class, &n}, func() error { out[class] = n; return nil }); err != nil { - return nil, unavailable(err) + return nil, cas.Down(err) } return out, nil } @@ -304,16 +299,7 @@ func (s *Source) fetch(name string) { } func (s *Source) lookupNow(ctx context.Context, name string) (tenants.TenantRecord, bool, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - r, err := scanRecord(s.pool.QueryRow(qctx, `SELECT name, token_sha256, max_claims, egress_class FROM sandboxd_tenants WHERE name = $1`, name)) - switch { - case errors.Is(err, pgx.ErrNoRows): - return tenants.TenantRecord{}, false, nil - case err != nil: - return tenants.TenantRecord{}, false, unavailable(err) - } - return r, true, nil + return cas.One(ctx, s.pool, scanRecord, `SELECT name, token_sha256, max_claims, egress_class FROM sandboxd_tenants WHERE name = $1`, name) } // write runs f in one transaction that notifies every node to evict key, then evicts it here at once. @@ -332,7 +318,7 @@ func (s *Source) write(ctx context.Context, key string, f func(tx pgx.Tx) error) if errors.Is(err, tenants.ErrInvalid) || errors.Is(err, tenants.ErrUnknown) { return err } - return unavailable(err) + return cas.Down(err) } s.evict(key) return nil @@ -420,10 +406,3 @@ func scanRecord(row pgx.Row) (tenants.TenantRecord, error) { r.TokenSHA256 = hex.EncodeToString(sum) return r, nil } - -func unavailable(err error) error { - if err == nil { - return nil - } - return fmt.Errorf("%w: %w", tenants.ErrUnavailable, err) -} diff --git a/sandboxd/tunnet/activity_test.go b/sandboxd/tunnet/activity_test.go index b88faef28..f7bbe2c72 100644 --- a/sandboxd/tunnet/activity_test.go +++ b/sandboxd/tunnet/activity_test.go @@ -8,7 +8,7 @@ import ( func TestOnlyAdmittedTrafficTouchesClaim(t *testing.T) { u := &fakeUnder{out: make(chan sent, 64)} - r := New(u, &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1450, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) + r := New(u, testOverlay(1450)) touches := 0 r.Arm(Claim{ ID: "c1", Addr: claimAddr, diff --git a/sandboxd/tunnet/addrs.go b/sandboxd/tunnet/addrs.go index 38bd075f9..177947db2 100644 --- a/sandboxd/tunnet/addrs.go +++ b/sandboxd/tunnet/addrs.go @@ -2,6 +2,7 @@ package tunnet import ( "bytes" + "cmp" "context" "encoding/binary" "errors" @@ -142,9 +143,7 @@ func (l *Local) Release(_ context.Context, claimID string, addr netip.Addr) erro return fmt.Errorf("open %s: %w", freedFile, err) } _, err = f.Write(appendFreed(nil, addr, now)) - if closeErr := f.Close(); err == nil { - err = closeErr - } + err = cmp.Or(err, f.Close()) if err != nil { return fmt.Errorf("append %s: %w", freedFile, err) } diff --git a/sandboxd/tunnet/bench_test.go b/sandboxd/tunnet/bench_test.go index d727c2b8e..3f304348c 100644 --- a/sandboxd/tunnet/bench_test.go +++ b/sandboxd/tunnet/bench_test.go @@ -53,7 +53,7 @@ func BenchmarkDeliver(b *testing.B) { } func benchLink() *link { - r := New(nopUnder{}, &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1450, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) + r := New(nopUnder{}, testOverlay(1450)) c := Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }} r.Arm(c) a := r.claims[claimAddr] diff --git a/sandboxd/tunnet/packet.go b/sandboxd/tunnet/packet.go index 3c6eb1c8f..8bd8c7755 100644 --- a/sandboxd/tunnet/packet.go +++ b/sandboxd/tunnet/packet.go @@ -26,9 +26,7 @@ const ( OffloadTSO6 = 0x04 OffloadTSOECN = 0x08 OffloadUSO4 = 0x20 - OffloadUSO6 = 0x40 - // IPv4 protocols the tun path carries. ProtoICMP = 1 ProtoTCP = 6 ProtoUDP = 17 @@ -260,13 +258,10 @@ func rewrite(frame []byte, at int, to [4]byte) { // adjust moves a ones'-complement sum from one address to another (RFC 1624). func adjust(sum uint16, from, to [4]byte) uint16 { - s := uint32(sum) + s := uint64(sum) for i := 0; i < 4; i += 2 { - s += uint32(^binary.BigEndian.Uint16(from[i:])) - s += uint32(binary.BigEndian.Uint16(to[i:])) + s += uint64(^binary.BigEndian.Uint16(from[i:])) + s += uint64(binary.BigEndian.Uint16(to[i:])) } - for s>>16 != 0 { - s = s&0xffff + s>>16 - } - return uint16(s) + return fold(s) } diff --git a/sandboxd/tunnet/pg/pg.go b/sandboxd/tunnet/pg/pg.go index 4feef9f68..d82f4768b 100644 --- a/sandboxd/tunnet/pg/pg.go +++ b/sandboxd/tunnet/pg/pg.go @@ -60,7 +60,9 @@ func (s *Store) Assign(ctx context.Context, claimID string) (netip.Addr, error) hosts := int64(1)<<(32-s.subnet.Bits()) - 2 for range assignTries { var addr string - err := s.scan(ctx, &addr, assignSQL, s.cell, claimID, s.subnet.Addr().String(), hosts, s.reuse.Seconds()) + qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) + err := s.pool.QueryRow(qctx, assignSQL, s.cell, claimID, s.subnet.Addr().String(), hosts, s.reuse.Seconds()).Scan(&addr) + cancel() switch { case err == nil: return netip.ParseAddr(addr) @@ -73,29 +75,19 @@ func (s *Store) Assign(ctx context.Context, claimID string) (netip.Addr, error) // Release stamps addr released; the stamp starts its reuse window. func (s *Store) Release(ctx context.Context, claimID string, addr netip.Addr) error { - return s.exec(ctx, `UPDATE sandboxd_tun_addrs SET released_at = now() - WHERE cell = $1 AND address = $2 AND claim = $3 AND released_at IS NULL`, s.cell, addr.String(), claimID) -} - -func (s *Store) Close() { - s.pool.Close() -} - -func (s *Store) scan(ctx context.Context, dest any, sql string, args ...any) error { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - return s.pool.QueryRow(qctx, sql, args...).Scan(dest) -} - -func (s *Store) exec(ctx context.Context, sql string, args ...any) error { qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) defer cancel() - if _, err := s.pool.Exec(qctx, sql, args...); err != nil { + if _, err := s.pool.Exec(qctx, `UPDATE sandboxd_tun_addrs SET released_at = now() + WHERE cell = $1 AND address = $2 AND claim = $3 AND released_at IS NULL`, s.cell, addr.String(), claimID); err != nil { return down(err) } return nil } +func (s *Store) Close() { + s.pool.Close() +} + func down(err error) error { return fmt.Errorf("tun address store: %w", err) } diff --git a/sandboxd/tunnet/router.go b/sandboxd/tunnet/router.go index 7ac357daa..9bdcd983a 100644 --- a/sandboxd/tunnet/router.go +++ b/sandboxd/tunnet/router.go @@ -19,7 +19,6 @@ import ( ) const ( - // Drop reasons, one counter each. DropNone Drop = iota DropMalformed DropNotIPv4 diff --git a/sandboxd/tunnet/router_test.go b/sandboxd/tunnet/router_test.go index be92f65c9..6b10bd5da 100644 --- a/sandboxd/tunnet/router_test.go +++ b/sandboxd/tunnet/router_test.go @@ -2,7 +2,6 @@ package tunnet import ( "bytes" - "context" "encoding/binary" "errors" "io" @@ -21,7 +20,7 @@ var ( ) func TestRouterAnnouncesRewritesAndPassesOnlyReplies(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) var records []FlowRecord var mu sync.Mutex r.Arm(Claim{ @@ -66,7 +65,7 @@ func TestRouterAnnouncesRewritesAndPassesOnlyReplies(t *testing.T) { } func TestRouterAnswersARPOnlyForArmedClaims(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr}) u.sent(t) asker := netip.MustParseAddr("100.80.0.1") @@ -84,14 +83,21 @@ func TestRouterAnswersARPOnlyForArmedClaims(t *testing.T) { r.Disarm(claimAddr) b = appendHeader(nil, 7, broadcast, exitMAC, etherARP) u.inject(t, appendARP(b, arpReq, exitMAC, asker, [6]byte{}, claimAddr)) - time.Sleep(50 * time.Millisecond) + sentinel := netip.MustParseAddr("100.80.0.8") + r.Arm(Claim{ID: "c2", Addr: sentinel}) + u.sent(t) + b = appendHeader(nil, 7, broadcast, exitMAC, etherARP) + u.inject(t, appendARP(b, arpReq, exitMAC, asker, [6]byte{}, sentinel)) + if inner, _, _ := u.sent(t); netip.AddrFrom4([4]byte(inner[14:18])) != sentinel { + t.Fatalf("first answer after the disarm announces %x, want only the sentinel's", inner[14:18]) + } if len(u.out) != 0 { t.Error("a disarmed claim, or a sender that is no exit, got an ARP answer") } } func TestRouterSegmentsGSOIntoOneBatch(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -109,7 +115,7 @@ func TestRouterSegmentsGSOIntoOneBatch(t *testing.T) { } func TestRouterRefusesUnarmedAndDisabledAndDisarmEndsStream(t *testing.T) { - _, r := newRouter() + _, r := newRouter(t) if h := guestHello(t, serve(t, r, claimAddr)); h.Flags&FlagOn != 0 { t.Error("an unarmed address got on") } @@ -132,7 +138,7 @@ func TestRouterRefusesUnarmedAndDisabledAndDisarmEndsStream(t *testing.T) { } func TestRouterQuotaStopsTraffic(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) used := new(atomic.Int64) r.Arm(Claim{ID: "c1", Addr: claimAddr, Used: used, Limits: Limits{QuotaBytes: 100}, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) @@ -150,7 +156,7 @@ func TestRouterQuotaStopsTraffic(t *testing.T) { } func TestRouterChargesNothingForAFrameWithNoExit(t *testing.T) { - _, r := newRouter() + _, r := newRouter(t) used := new(atomic.Int64) r.Arm(Claim{ID: "c1", Addr: claimAddr, Profile: "nowhere", Used: used, Allow: func(netip.Addr, uint16) bool { return true }}) guest := serve(t, r, claimAddr) @@ -169,7 +175,7 @@ func TestRouterChargesNothingForAFrameWithNoExit(t *testing.T) { } func TestDeliverCoalescesInOrderSegmentsIntoOneGSOFrame(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -196,7 +202,7 @@ func TestDeliverCoalescesInOrderSegmentsIntoOneGSOFrame(t *testing.T) { } func TestRouterChargesSegmentsAndRefusesOversizedPackets(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Limits: Limits{PacketsPerSecond: 10}, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) go func() { @@ -220,7 +226,7 @@ func TestRouterChargesSegmentsAndRefusesOversizedPackets(t *testing.T) { } func TestRouterLimitsOutliveAReconnect(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Limits: Limits{NewFlowsPerSecond: 0.01}, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) first := serve(t, r, claimAddr) @@ -237,7 +243,7 @@ func TestRouterLimitsOutliveAReconnect(t *testing.T) { } func TestRouterHandsAnICMPErrorBackWithTheQuotedGuestAddress(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -253,7 +259,7 @@ func TestRouterHandsAnICMPErrorBackWithTheQuotedGuestAddress(t *testing.T) { } func TestRouterResumesAStreamOpenedBeforeTheArmButRefusesAnOpener(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(_ netip.Addr, port uint16) bool { return port == 443 }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -270,12 +276,12 @@ func TestRouterResumesAStreamOpenedBeforeTheArmButRefusesAnOpener(t *testing.T) } func TestRouterKeepsTheMTUAStreamWasToldUntilItsNextHello(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) guest := serve(t, r, claimAddr) guestHello(t, guest) - r.Reload(&Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1410, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) + r.Reload(testOverlay(1410)) u.sent(t) full := gsoFrame(ProtoUDP, 1412, 1412) full[0], full[1] = 0, gsoNone @@ -286,7 +292,7 @@ func TestRouterKeepsTheMTUAStreamWasToldUntilItsNextHello(t *testing.T) { } func TestEndFlowsRecordsLiveFlowsWithIPBytes(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) var mu sync.Mutex var records []FlowRecord r.Arm(Claim{ @@ -311,7 +317,7 @@ func TestEndFlowsRecordsLiveFlowsWithIPBytes(t *testing.T) { } func TestDeliverMergesOnlyForAGuestThatOfferedTSO(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) host, guest := net.Pipe() @@ -333,7 +339,7 @@ func TestDeliverMergesOnlyForAGuestThatOfferedTSO(t *testing.T) { } func TestRouterPointsAGuestICMPErrorAtTheClaimAddress(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -346,13 +352,19 @@ func TestRouterPointsAGuestICMPErrorAtTheClaimAddress(t *testing.T) { } } -func newRouter() (*fakeUnder, *Router) { - u := &fakeUnder{out: make(chan sent, 64), in: make(chan []sent, 16)} - r := New(u, &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1450, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) - go func() { _ = r.Run(context.Background()) }() +func newRouter(t *testing.T) (*fakeUnder, *Router) { + t.Helper() + u := &fakeUnder{out: make(chan sent, 64), in: make(chan []sent, 16), done: make(chan struct{})} + t.Cleanup(func() { close(u.done) }) + r := New(u, testOverlay(1450)) + go func() { _ = r.Run(t.Context()) }() return u, r } +func testOverlay(mtu uint16) *Overlay { + return &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: mtu, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}} +} + func serve(t *testing.T, r *Router, addr netip.Addr) net.Conn { t.Helper() host, guest := net.Pipe() @@ -461,6 +473,7 @@ type sent struct { type fakeUnder struct { out chan sent in chan []sent + done chan struct{} batches atomic.Int64 } @@ -475,10 +488,15 @@ func (u *fakeUnder) Send(b []byte, seg int, dst netip.AddrPort, _ uint64) error } func (u *fakeUnder) Recv(each func(d []byte, from netip.AddrPort)) error { - for _, s := range <-u.in { - each(s.d, s.to) + select { + case batch := <-u.in: + for _, s := range batch { + each(s.d, s.to) + } + return nil + case <-u.done: + return net.ErrClosed } - return nil } func (u *fakeUnder) inject(t *testing.T, d []byte) { diff --git a/sandboxd/tunnet/udp/exit_linux_test.go b/sandboxd/tunnet/udp/exit_linux_test.go index 674051b55..4b1753fe0 100644 --- a/sandboxd/tunnet/udp/exit_linux_test.go +++ b/sandboxd/tunnet/udp/exit_linux_test.go @@ -3,6 +3,7 @@ package udp_test import ( "bytes" "encoding/binary" + "fmt" "io" "net" "net/netip" @@ -48,54 +49,55 @@ func TestTrafficCrossesAKernelExit(t *testing.T) { go func() { _ = r.Serve(host, claimAddr) }() go pumpGuest(tun, guest) - inNetns(t, "inet", func() { + inNetns(t, "inet", func() error { l, err := net.Listen("tcp", "10.98.0.2:7000") if err != nil { - t.Fatal(err) + return err } c, err := net.ListenPacket("udp", "10.98.0.2:7001") if err != nil { - t.Fatal(err) + return err } go echoTCP(l) go echoUDP(c) + return nil }) - time.Sleep(300 * time.Millisecond) waitNeigh(t) - var got []byte - inNetns(t, "guest", func() { + inNetns(t, "guest", func() error { c, err := net.DialTimeout("tcp", "10.98.0.2:7000", 5*time.Second) if err != nil { - t.Log(run(t, "sh", "-c", "ip netns exec guest cat /proc/net/snmp | grep -E '^(Ip|Tcp|Udp):'; ip netns exec exit cat /proc/net/snmp | grep -E '^(Ip|Tcp):'; ip netns exec exit conntrack -L 2>&1 | head; ip -n exit -s link show vxlan0 | head -8")) - t.Fatalf("tcp through the exit: %v", err) + diag, _ := runOut("sh", "-c", "ip netns exec guest cat /proc/net/snmp | grep -E '^(Ip|Tcp|Udp):'; ip netns exec exit cat /proc/net/snmp | grep -E '^(Ip|Tcp):'; ip netns exec exit conntrack -L 2>&1 | head; ip -n exit -s link show vxlan0 | head -8") + t.Log(diag) + return fmt.Errorf("tcp through the exit: %w", err) } defer func() { _ = c.Close() }() start := time.Now() go func() { _, _ = c.Write(bytes.Repeat([]byte{0x5a}, bulk)); _ = c.(*net.TCPConn).CloseWrite() }() - got, err = io.ReadAll(c) + got, err := io.ReadAll(c) if err != nil || len(got) != bulk { - t.Fatalf("tcp echo %d bytes, err %v", len(got), err) + return fmt.Errorf("tcp echo %d bytes, err %v", len(got), err) } t.Logf("tcp echo %d MiB both ways in %v", bulk>>20, time.Since(start)) u, err := net.Dial("udp", "10.98.0.2:7001") if err != nil { - t.Fatal(err) + return err } defer func() { _ = u.Close() }() _ = u.SetReadDeadline(time.Now().Add(3 * time.Second)) if _, err = u.Write([]byte("ping")); err != nil { - t.Fatal(err) + return err } b := make([]byte, 64) n, err := u.Read(b) if err != nil || !strings.HasPrefix(string(b[:n]), "10.98.0.1") { - t.Fatalf("udp echo %q %v, want the exit's masquerade address", b[:n], err) + return fmt.Errorf("udp echo %q %v, want the exit's masquerade address", b[:n], err) } other, _ := net.DialTimeout("tcp", otherAddr.String()+":7000", 500*time.Millisecond) if other != nil { t.Error("a claim reached another claim's overlay address") } + return nil }) ct := run(t, "ip", "netns", "exec", "exit", "conntrack", "-L", "-s", claimAddr.String()) if !strings.Contains(ct, "src="+claimAddr.String()) { @@ -143,20 +145,21 @@ NFT func guestTun(t *testing.T) *os.File { t.Helper() var f *os.File - inNetns(t, "guest", func() { + inNetns(t, "guest", func() error { fd, err := unix.Open("/dev/net/tun", unix.O_RDWR|unix.O_CLOEXEC, 0) if err != nil { - t.Fatal(err) + return err } ifr, _ := unix.NewIfreq("tun0") ifr.SetUint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR) if err = unix.IoctlIfreq(fd, unix.TUNSETIFF, ifr); err != nil { - t.Fatal(err) + return err } if err = unix.IoctlSetInt(fd, unix.TUNSETOFFLOAD, tunnet.OffloadCSUM|tunnet.OffloadTSO4|tunnet.OffloadTSO6|tunnet.OffloadTSOECN); err != nil { - t.Fatal(err) + return err } f = os.NewFile(uintptr(fd), "tun0") + return nil }) run(t, "sh", "-c", "ip -n guest addr add 100.64.0.2/30 dev tun0 && ip -n guest link set tun0 mtu 1450 up && ip -n guest route add default via 100.64.0.1") return f @@ -230,32 +233,38 @@ func waitNeigh(t *testing.T) { } // inNetns runs fn on a thread inside netns name; sockets fn opens stay in that netns. -func inNetns(t *testing.T, name string, fn func()) { +func inNetns(t *testing.T, name string, fn func() error) { t.Helper() - done := make(chan struct{}) + errc := make(chan error, 1) go func() { - defer close(done) runtime.LockOSThread() f, err := os.Open("/var/run/netns/" + name) if err != nil { - t.Error(err) + errc <- err return } defer func() { _ = f.Close() }() if err = unix.Setns(int(f.Fd()), unix.CLONE_NEWNET); err != nil { - t.Error(err) + errc <- err return } - fn() + errc <- fn() }() - <-done + if err := <-errc; err != nil { + t.Fatal(err) + } } func run(t *testing.T, name string, args ...string) string { t.Helper() - out, err := exec.Command(name, args...).CombinedOutput() + out, err := runOut(name, args...) if err != nil { t.Fatalf("%s %v: %v\n%s", name, args, err, out) } - return string(out) + return out +} + +func runOut(name string, args ...string) (string, error) { + out, err := exec.Command(name, args...).CombinedOutput() + return string(out), err } diff --git a/sandboxd/types/bench_test.go b/sandboxd/types/bench_test.go index 279e571e4..5f81488b8 100644 --- a/sandboxd/types/bench_test.go +++ b/sandboxd/types/bench_test.go @@ -1,10 +1,6 @@ package types -import ( - "sync" - "testing" - "time" -) +import "testing" func BenchmarkSandboxTouch(b *testing.B) { sb := &Sandbox{} @@ -14,18 +10,3 @@ func BenchmarkSandboxTouch(b *testing.B) { } }) } - -func BenchmarkSandboxTouchMutex(b *testing.B) { - var ( - mu sync.Mutex - t time.Time - ) - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - mu.Lock() - t = time.Now() - mu.Unlock() - } - }) - _ = t -} diff --git a/sandboxd/types/volume_test.go b/sandboxd/types/volume_test.go index 79947d597..6fde38cc7 100644 --- a/sandboxd/types/volume_test.go +++ b/sandboxd/types/volume_test.go @@ -45,16 +45,7 @@ func TestValidateVolumes(t *testing.T) { if !slices.Equal(volumes, wantInput) { t.Errorf("input mutated to %v, want %v", volumes, wantInput) } - if got, err := ValidateVolumes([]Volume{ - {Name: "a"}, - {Name: "b"}, - {Name: "c"}, - {Name: "d"}, - {Name: "e"}, - {Name: "f"}, - {Name: "g"}, - {Name: "h"}, - }); err != nil || len(got) != MaxClaimVolumes { + if got, err := ValidateVolumes(namedVolumes(MaxClaimVolumes)); err != nil || len(got) != MaxClaimVolumes { t.Errorf("ValidateVolumes at limit: got=%v err=%v", got, err) } @@ -62,7 +53,7 @@ func TestValidateVolumes(t *testing.T) { name string volumes []Volume }{ - {"too-many", []Volume{{Name: "a"}, {Name: "b"}, {Name: "c"}, {Name: "d"}, {Name: "e"}, {Name: "f"}, {Name: "g"}, {Name: "h"}, {Name: "i"}}}, + {"too-many", namedVolumes(MaxClaimVolumes + 1)}, {"invalid-name", []Volume{{Name: "bad/name"}}}, {"duplicate-name", []Volume{{Name: "dataset"}, {Name: "dataset", Mount: "/other"}}}, {"relative-mount", []Volume{{Name: "dataset", Mount: "data"}}}, @@ -167,3 +158,11 @@ func TestValidateMixedVolumesPreservesPerEntryIntent(t *testing.T) { t.Fatalf("second validation changed mixed intent: %+v, %v", again, err) } } + +func namedVolumes(n int) []Volume { + volumes := make([]Volume, n) + for i := range volumes { + volumes[i].Name = string(rune('a' + i)) + } + return volumes +} diff --git a/sandboxd/usage/ship.go b/sandboxd/usage/ship.go index 207941d3e..348d1ff83 100644 --- a/sandboxd/usage/ship.go +++ b/sandboxd/usage/ship.go @@ -145,7 +145,6 @@ func (s *shipper) locate(ctx context.Context, live, backup *os.File) (*os.File, return found, s.at.end } -// read decodes up to shipBatch lines of files, the first from off, into s.node's next records. func (s *shipper) read(ctx context.Context, files []*os.File, off int64) ([]Record, anchor, int) { var ( recs []Record diff --git a/sdk/go/README.md b/sdk/go/README.md index 3374ada84..3b7c2e18b 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -26,7 +26,7 @@ out, _ := sb.Exec(ctx, "echo", "hello") - `upgrade.go` / `keepalive.go` / `endpoint.go` — the relay upgrade, idle connection reuse, endpoint and TLS parsing - `silkd/` — the conn/stream layer over the relay; the frame types live in `protocol/wire`, whose tests round-trip `protocol/wire/fixtures/` (drift - against the Rust guest fails CI); `silkdtest/` is an in-process fake guest + against the Rust guest fails CI); `silkd/silkdtest/` is an in-process fake guest for consumers' tests Module path `github.com/cocoonstack/sandbox/sdk/go`; no third-party diff --git a/sdk/go/checkpoint.go b/sdk/go/checkpoint.go index 9c09ba01a..9c243b896 100644 --- a/sdk/go/checkpoint.go +++ b/sdk/go/checkpoint.go @@ -49,10 +49,7 @@ type Checkpoint struct { // New claims a sandbox branched from the checkpoint, following an owner redirect. func (ck *Checkpoint) New(ctx context.Context, opts ...Option) (*Sandbox, error) { - var claim claimRequest - for _, opt := range opts { - opt(&claim) - } + claim := newClaim("", opts) if len(claim.Volumes) > 0 || claim.ClaimRef != "" { return nil, errors.New("checkpoint claims do not accept WithVolumes or WithClaimRef") } diff --git a/sdk/go/checkpoint_test.go b/sdk/go/checkpoint_test.go index 18ec77867..225e543be 100644 --- a/sdk/go/checkpoint_test.go +++ b/sdk/go/checkpoint_test.go @@ -23,7 +23,7 @@ func TestCheckpointNewFollowsRedirect(t *testing.T) { nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { nodeACalls++ _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, + Redirect: []string{addrOf(nodeB)}, }) })) t.Cleanup(nodeA.Close) @@ -37,7 +37,7 @@ func TestCheckpointNewFollowsRedirect(t *testing.T) { if sb.ID != "sb_ck1" { t.Errorf("id %q, want sb_ck1 (claimed at the redirect target)", sb.ID) } - if sb.owner != strings.TrimPrefix(nodeB.URL, "http://") { + if sb.owner != addrOf(nodeB) { t.Errorf("owner %q, want node B", sb.owner) } if nodeACalls != 1 { @@ -78,7 +78,7 @@ func TestCheckpointNewRedirectAllCandidatesFail(t *testing.T) { entryCalls++ if entryCalls == 1 { _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(broken.URL, "http://")}, + Redirect: []string{addrOf(broken)}, }) return } @@ -114,7 +114,7 @@ func TestCheckpointNewRedirectFallbackHeals(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { entryCalls++ if entryCalls == 1 { - _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(busy.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{addrOf(busy)}}) return } var req checkpointClaimRequest @@ -165,7 +165,7 @@ func TestCheckpointNewSecondLevelRedirectFails(t *testing.T) { nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, + Redirect: []string{addrOf(nodeB)}, }) })) t.Cleanup(nodeA.Close) diff --git a/sdk/go/client.go b/sdk/go/client.go index 816672f7f..7636f6a91 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/url" "slices" @@ -19,61 +20,15 @@ import ( ) const ( - idleConnsPerHost = 100 - - templateQueryParam = "template" - netQueryParam = "net" - sizeQueryParam = "size" + idleConnsPerHost = 100 noRedirectQueryParam = "no_redirect" ) +var errRedirectedAgain = errors.New("redirected again despite no_redirect") + // ClientOption configures Connect. type ClientOption func(*Client) -type claimEncoder func(noRedirect, requirePromoted bool) ([]byte, error) - -type claimPoster func(addr string, body []byte) (claimResponse, error) - -type volumeListResponse struct { - Volumes []VolumeInfo `json:"volumes"` -} - -type forkRequest struct { - Token string `json:"token"` - Count int `json:"count"` - TTLSeconds int `json:"ttl_seconds,omitzero"` -} - -type forkResponse struct { - Children []claimResponse `json:"children"` -} - -type promoteRequest struct { - Token string `json:"token"` - Template string `json:"template"` -} - -type promoteResponse struct { - Key struct { - Template string `json:"template"` - Net string `json:"net"` - Size string `json:"size"` - } `json:"key"` - ContentDigest string `json:"content_digest"` -} - -type renewRequest struct { - TTLSeconds int `json:"ttl_seconds,omitzero"` -} - -type renewResponse struct { - Deadline time.Time `json:"deadline"` -} - -type errorResponse struct { - Error string `json:"error"` -} - // APIError is a non-2xx control-plane reply. type APIError struct { Verb string @@ -88,6 +43,18 @@ func (e *APIError) Error() string { return fmt.Sprintf("%s: http %d", e.Verb, e.Status) } +type claimEncoder func(noRedirect, requirePromoted bool) ([]byte, error) + +type claimPoster func(addr string, body []byte) (claimResponse, error) + +type volumeListResponse struct { + Volumes []VolumeInfo `json:"volumes"` +} + +type errorResponse struct { + Error string `json:"error"` +} + type claimRequest struct { Template string `json:"template"` Net string `json:"net,omitempty"` @@ -132,14 +99,13 @@ type Client struct { keepAlive time.Duration } -// Connect returns a client for a sandboxd node; a comma-separated seed list keeps only its first address. +// Connect returns a client for a sandboxd node. func Connect(addr string, opts ...ClientOption) (*Client, error) { - first, _, _ := strings.Cut(addr, ",") - first = strings.TrimSpace(first) - if first == "" { + addr = strings.TrimSpace(addr) + if addr == "" { return nil, errors.New("empty sandboxd address") } - u, err := endpointURL(first, "http") + u, err := endpointURL(addr, "http") if err != nil { return nil, err } @@ -155,10 +121,7 @@ func Connect(addr string, opts ...ClientOption) (*Client, error) { // New claims a sandbox for template. func (c *Client) New(ctx context.Context, template string, opts ...Option) (*Sandbox, error) { - claim := claimRequest{Template: template} - for _, opt := range opts { - opt(&claim) - } + claim := newClaim(template, opts) addr, cr, err := claimFollow(c.addr, "claim", func(noRedirect, requirePromoted bool) ([]byte, error) { claim.NoRedirect, claim.RequirePromoted = noRedirect, requirePromoted return encodeBody("claim", claim) @@ -182,13 +145,16 @@ func (c *Client) Volumes(ctx context.Context) ([]VolumeInfo, error) { // Lookup finds the sandbox owner using its id and token. func (c *Client) Lookup(ctx context.Context, id, token string) (*Sandbox, error) { - if owner, err := c.ownerAt(ctx, c.addr, id, token); err == nil { + probe := func(ctx context.Context, addr string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, peersTimeout) + defer cancel() + return c.ownerAt(ctx, addr, id, token) + } + if owner, err := probe(ctx, c.addr); err == nil { return &Sandbox{ID: id, token: token, c: c, owner: owner}, nil } addrs, peersErr := c.peersOrErr(ctx) - owner, ok := scatter(ctx, addrs, func(ctx context.Context, addr string) (string, error) { - return c.ownerAt(ctx, addr, id, token) - }) + owner, ok := scatter(ctx, addrs, probe) if !ok { return nil, errors.Join(fmt.Errorf("lookup %s: no owner found", id), peersErr) } @@ -202,11 +168,8 @@ func (c *Client) Attach(ownerAddr, id, token string) *Sandbox { // DeleteTemplate removes a promoted template by name. func (c *Client) DeleteTemplate(ctx context.Context, template string, opts ...Option) error { - claim := claimRequest{Template: template} - for _, opt := range opts { - opt(&claim) - } - u := url.Values{templateQueryParam: {claim.Template}, netQueryParam: {claim.Net}, sizeQueryParam: {claim.Size}} + claim := newClaim(template, opts) + u := templateQuery(claim.Template, claim.Net, claim.Size) redirect, err := c.deleteTemplates(ctx, c.addr, u) if err != nil || len(redirect) == 0 { return err @@ -336,8 +299,7 @@ func retryTransient(err error) bool { } switch he.Status { case http.StatusUnauthorized, http.StatusNotFound, http.StatusTooManyRequests, - http.StatusServiceUnavailable, http.StatusInternalServerError, - http.StatusBadGateway, http.StatusGatewayTimeout: + http.StatusServiceUnavailable, http.StatusInternalServerError: return true default: return false @@ -375,7 +337,7 @@ func redirectFallback(origin string, candidates []string, claimAt func(addr stri return claimResponse{}, err } if len(cr.Redirect) > 0 { - return claimResponse{}, fmt.Errorf("%s redirected again despite no_redirect", target) + return claimResponse{}, fmt.Errorf("%s %w", target, errRedirectedAgain) } return cr, nil } @@ -385,6 +347,9 @@ func redirectFallback(origin string, candidates []string, claimAt func(addr stri if err == nil { return addr, cr, nil } + if !claimRefused(err) { + return "", claimResponse{}, fmt.Errorf("redirect target %s: %w", addr, err) + } lastErr = err } if !retryTransient(lastErr) { @@ -397,6 +362,16 @@ func redirectFallback(origin string, candidates []string, claimAt func(addr stri return origin, cr, nil } +func claimRefused(err error) bool { + if he, ok := errors.AsType[*APIError](err); ok { + return he.Status != http.StatusBadGateway && he.Status != http.StatusGatewayTimeout + } + if op, ok := errors.AsType[*net.OpError](err); ok && op.Op == "dial" { + return true + } + return errors.Is(err, errRedirectedAgain) +} + func scatter[T any](ctx context.Context, addrs []string, probe func(ctx context.Context, addr string) (T, error)) (result T, ok bool) { scatterCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -428,3 +403,15 @@ func apiError(verb string, resp *http.Response) error { _, _ = io.Copy(io.Discard, resp.Body) return &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} } + +func newClaim(template string, opts []Option) claimRequest { + claim := claimRequest{Template: template} + for _, opt := range opts { + opt(&claim) + } + return claim +} + +func templateQuery(name, net, size string) url.Values { + return url.Values{"template": {name}, "net": {net}, "size": {size}} +} diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 6e14f1919..4a44bf5e2 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -25,9 +25,9 @@ func TestConnectAddress(t *testing.T) { ok bool }{ {"single", "n1:7777", "n1:7777", true}, - {"seed list uses first", "n1:7777, n2:7777", "n1:7777", true}, + {"seed list refused", "n1:7777, n2:7777", "", false}, {"empty", "", "", false}, - {"only commas", " , ", "", false}, + {"blank", " ", "", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -137,7 +137,7 @@ func TestCloseReleasesASandboxThatMoved(t *testing.T) { adopter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v1/sandboxes/sb_9/owner": - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(adopter.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(adopter)}) case "/v1/sandboxes/sb_9/release": released.Store(true) w.WriteHeader(http.StatusNoContent) @@ -146,7 +146,7 @@ func TestCloseReleasesASandboxThatMoved(t *testing.T) { t.Cleanup(adopter.Close) old := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/peers" { - _ = json.MarshalWrite(w, map[string]any{"peers": []string{strings.TrimPrefix(adopter.URL, "http://")}}) + _ = json.MarshalWrite(w, map[string]any{"peers": []string{addrOf(adopter)}}) return } w.WriteHeader(http.StatusNotFound) @@ -183,6 +183,27 @@ func TestAMovedSandboxRetriesAControlCallAtItsNewOwner(t *testing.T) { } } +func TestAMovedSandboxRetriesAStreamCallAtItsNewOwner(t *testing.T) { + for name, call := range map[string]func(*Sandbox) error{ + "write file": func(sb *Sandbox) error { return sb.WriteFile(t.Context(), "/f", []byte("x"), nil) }, + "watch": func(sb *Sandbox) error { + w, err := sb.Watch(t.Context(), "/", false) + if err == nil { + err = w.Close() + } + return err + }, + } { + sb, adopter := movedSandbox(t) + if err := call(sb); err != nil { + t.Errorf("%s after the move: %v", name, err) + } + if got := sb.Owner(); got != adopter { + t.Errorf("%s: owner %s, want the adopter %s", name, got, adopter) + } + } +} + func TestAMovedSandboxDoesNotReplayARunThatStreamedStdin(t *testing.T) { sb, adopter := movedSandbox(t) if _, err := sb.Run(t.Context(), Cmd{Argv: []string{"cat"}, Stdin: strings.NewReader("once")}); err == nil { @@ -218,10 +239,10 @@ func TestAWriterErrorTaggedForThisSandboxIsNotReplayed(t *testing.T) { }) owner := newAgentServer(t, fake.ServeConn) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(other.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(other)}) })) t.Cleanup(entry.Close) - sb := &Sandbox{ID: "sb_1", c: testClient(t, entry), token: "tok", owner: strings.TrimPrefix(owner.URL, "http://")} + sb := &Sandbox{ID: "sb_1", c: testClient(t, entry), token: "tok", owner: addrOf(owner)} t.Cleanup(sb.pool.drain) tagged := writerFunc(func([]byte) (int, error) { return 0, &ownerGoneError{err: syscall.ECONNREFUSED, id: sb.ID, addr: sb.Owner()} @@ -247,11 +268,11 @@ func TestAnotherSandboxsErrorInAWriterIsNotReplayed(t *testing.T) { w.WriteHeader(http.StatusNotFound) return } - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(other.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(other)}) })) t.Cleanup(entry.Close) c := testClient(t, entry) - sb := &Sandbox{ID: "sb_1", c: c, token: "tok", owner: strings.TrimPrefix(owner.URL, "http://")} + sb := &Sandbox{ID: "sb_1", c: c, token: "tok", owner: addrOf(owner)} gone := &Sandbox{ID: "sb_2", c: c, token: "tok", owner: retiredAddr(t)} t.Cleanup(sb.pool.drain) writeElsewhere := writerFunc(func([]byte) (int, error) { @@ -284,7 +305,7 @@ func TestCloseFollowsAMoveOffARetiredNode(t *testing.T) { adopter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v1/sandboxes/sb_9/owner": - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(adopter.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(adopter)}) case "/v1/sandboxes/sb_9/release": released.Store(true) w.WriteHeader(http.StatusNoContent) @@ -317,17 +338,12 @@ func TestNewFollowsRedirect(t *testing.T) { var nodeB *httptest.Server nodeB = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, claimResponse{ - ID: "sb_2", Token: "tok", OwnerAddr: strings.TrimPrefix(nodeB.URL, "http://"), + ID: "sb_2", Token: "tok", OwnerAddr: addrOf(nodeB), }) })) t.Cleanup(nodeB.Close) - nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, - }) - })) - t.Cleanup(nodeA.Close) + nodeA := redirectEntry(t, addrOf(nodeB)) sb, err := testClient(t, nodeA).New(t.Context(), "rt:24.04") if err != nil { @@ -336,7 +352,7 @@ func TestNewFollowsRedirect(t *testing.T) { if sb.ID != "sb_2" { t.Errorf("id %q, want sb_2 (claimed at the redirect target)", sb.ID) } - if sb.owner != strings.TrimPrefix(nodeB.URL, "http://") { + if sb.owner != addrOf(nodeB) { t.Errorf("owner %q, want node B", sb.owner) } } @@ -354,12 +370,7 @@ func TestDeleteTemplateFollowsRedirect(t *testing.T) { })) t.Cleanup(nodeB.Close) - nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, - }) - })) - t.Cleanup(nodeA.Close) + nodeA := redirectEntry(t, addrOf(nodeB)) if err := testClient(t, nodeA).DeleteTemplate(t.Context(), "tpl"); err != nil { t.Fatalf("DeleteTemplate: %v", err) @@ -373,31 +384,11 @@ func TestDeleteTemplateFollowsRedirect(t *testing.T) { } func TestDeleteTemplateRetryPolicy(t *testing.T) { - status := func(code int, hit *bool) *httptest.Server { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - *hit = true - w.WriteHeader(code) - })) - t.Cleanup(s.Close) - return s - } - entry := func(candidates ...*httptest.Server) *httptest.Server { - addrs := make([]string, len(candidates)) - for i, c := range candidates { - addrs[i] = strings.TrimPrefix(c.URL, "http://") - } - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{Redirect: addrs}) - })) - t.Cleanup(s.Close) - return s - } - t.Run("miss walks to the next owner", func(t *testing.T) { var missHit, ownerHit bool - miss := status(http.StatusNotFound, &missHit) - owner := status(http.StatusNoContent, &ownerHit) - if err := testClient(t, entry(miss, owner)).DeleteTemplate(t.Context(), "tpl"); err != nil { + miss := statusServer(t, http.StatusNotFound, &missHit) + owner := statusServer(t, http.StatusNoContent, &ownerHit) + if err := testClient(t, redirectEntry(t, addrOf(miss), addrOf(owner))).DeleteTemplate(t.Context(), "tpl"); err != nil { t.Fatalf("DeleteTemplate: %v", err) } if !missHit || !ownerHit { @@ -406,9 +397,9 @@ func TestDeleteTemplateRetryPolicy(t *testing.T) { }) t.Run("served error stops the walk", func(t *testing.T) { var brokenHit, ownerHit bool - broken := status(http.StatusInternalServerError, &brokenHit) - owner := status(http.StatusNoContent, &ownerHit) - err := testClient(t, entry(broken, owner)).DeleteTemplate(t.Context(), "tpl") + broken := statusServer(t, http.StatusInternalServerError, &brokenHit) + owner := statusServer(t, http.StatusNoContent, &ownerHit) + err := testClient(t, redirectEntry(t, addrOf(broken), addrOf(owner))).DeleteTemplate(t.Context(), "tpl") if err == nil || !strings.Contains(err.Error(), "http 500") { t.Fatalf("err %v, want the 500 surfaced", err) } @@ -428,12 +419,7 @@ func TestRedirectSetsNoRedirect(t *testing.T) { })) t.Cleanup(target.Close) - entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(target.URL, "http://")}, - }) - })) - t.Cleanup(entry.Close) + entry := redirectEntry(t, addrOf(target)) sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") if err != nil { @@ -447,94 +433,112 @@ func TestRedirectSetsNoRedirect(t *testing.T) { } } -func TestRedirectTriesAllCandidates(t *testing.T) { - good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ID: "sb_4", Token: "t"}) - })) - t.Cleanup(good.Close) - - entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{"127.0.0.1:1", strings.TrimPrefix(good.URL, "http://")}, +func TestRedirectWalksPastACandidateThatRefusedTheClaim(t *testing.T) { + for _, tt := range []struct { + name string + first func(*testing.T) string + }{ + {"unreachable", func(*testing.T) string { return "127.0.0.1:1" }}, + {"forbidden", func(t *testing.T) string { return addrOf(statusServer(t, http.StatusForbidden, nil)) }}, + } { + t.Run(tt.name, func(t *testing.T) { + good := serveFunc(t, func(w http.ResponseWriter, _ *http.Request) { + _ = json.MarshalWrite(w, claimResponse{ID: "sb_5", Token: "t"}) + }) + entry := redirectEntry(t, tt.first(t), addrOf(good)) + + sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") + if err != nil { + t.Fatalf("New: %v", err) + } + if sb.ID != "sb_5" { + t.Errorf("id %q, want sb_5 (second candidate, reached despite the first's refusal)", sb.ID) + } }) - })) - t.Cleanup(entry.Close) - - sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") - if err != nil { - t.Fatalf("New: %v", err) - } - if sb.ID != "sb_4" { - t.Errorf("id %q, want sb_4 (second candidate)", sb.ID) } } -func TestRedirectCandidateDefinitiveErrorStillTriesNextCandidate(t *testing.T) { - forbidden := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusForbidden) +func TestRedirectStopsWhenACandidateMayHaveClaimed(t *testing.T) { + var goodCalls, entryCalls atomic.Int32 + lost := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + conn, _, err := http.NewResponseController(w).Hijack() + if err == nil { + _ = conn.Close() + } })) - t.Cleanup(forbidden.Close) + t.Cleanup(lost.Close) good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ID: "sb_5", Token: "t"}) + goodCalls.Add(1) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_6", Token: "t"}) })) t.Cleanup(good.Close) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + entryCalls.Add(1) _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(forbidden.URL, "http://"), strings.TrimPrefix(good.URL, "http://")}, + Redirect: []string{addrOf(lost), addrOf(good)}, }) })) t.Cleanup(entry.Close) - sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") - if err != nil { - t.Fatalf("New: %v", err) + if _, err := testClient(t, entry).New(t.Context(), "rt:24.04"); err == nil { + t.Fatal("New succeeded after a candidate dropped the reply to a delivered claim") } - if sb.ID != "sb_5" { - t.Errorf("id %q, want sb_5 (second candidate, reached despite the first's 403)", sb.ID) + if goodCalls.Load() != 0 || entryCalls.Load() != 1 { + t.Errorf("good=%d entry=%d calls, want 0 and 1: a delivered claim is never sent again", goodCalls.Load(), entryCalls.Load()) } } -func TestRedirectFallbackHeals(t *testing.T) { - var fullCalls int - full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - fullCalls++ - w.WriteHeader(http.StatusTooManyRequests) - })) - t.Cleanup(full.Close) - - var entryCalls int - entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - entryCalls++ - if entryCalls == 1 { - _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(full.URL, "http://")}}) - return - } - var req claimRequest - _ = json.UnmarshalRead(r.Body, &req) - if !req.NoRedirect { - t.Error("origin fallback did not carry no_redirect") - } - _ = json.MarshalWrite(w, claimResponse{ID: "sb_healed", Token: "t"}) - })) - t.Cleanup(entry.Close) +func TestRedirectFallbackProvisionsAtTheOrigin(t *testing.T) { + for _, tt := range []struct { + name string + code int + body string + }{ + {"candidate full", http.StatusTooManyRequests, ""}, + {"rotating token", http.StatusUnauthorized, "invalid api token"}, + } { + t.Run(tt.name, func(t *testing.T) { + var candidateCalls, entryCalls int + candidate := serveFunc(t, func(w http.ResponseWriter, _ *http.Request) { + candidateCalls++ + w.WriteHeader(tt.code) + if tt.body != "" { + _ = json.MarshalWrite(w, errorResponse{Error: tt.body}) + } + }) + entry := serveFunc(t, func(w http.ResponseWriter, r *http.Request) { + entryCalls++ + if entryCalls == 1 { + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{addrOf(candidate)}}) + return + } + var req claimRequest + _ = json.UnmarshalRead(r.Body, &req) + if !req.NoRedirect { + t.Error("origin fallback did not carry no_redirect") + } + _ = json.MarshalWrite(w, claimResponse{ID: "sb_origin", Token: "t"}) + }) - sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") - if err != nil { - t.Fatalf("New: %v", err) - } - if sb.ID != "sb_healed" { - t.Errorf("id %q, want sb_healed", sb.ID) - } - if sb.owner != strings.TrimPrefix(entry.URL, "http://") { - t.Errorf("owner %q, want the origin", sb.owner) - } - if entryCalls != 2 { - t.Errorf("origin called %d times, want 2 (initial + fallback)", entryCalls) - } - if fullCalls != 1 { - t.Errorf("candidate called %d times, want 1", fullCalls) + sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") + if err != nil { + t.Fatalf("New: %v", err) + } + if sb.ID != "sb_origin" { + t.Errorf("id %q, want sb_origin (provisioned at the origin)", sb.ID) + } + if sb.owner != addrOf(entry) { + t.Errorf("owner %q, want the origin", sb.owner) + } + if entryCalls != 2 { + t.Errorf("origin called %d times, want 2 (initial + fallback)", entryCalls) + } + if candidateCalls != 1 { + t.Errorf("candidate called %d times, want 1", candidateCalls) + } + }) } } @@ -551,7 +555,7 @@ func TestRedirectFallbackAlsoFails(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { entryCalls++ if entryCalls == 1 { - _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(full.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{addrOf(full)}}) return } w.WriteHeader(http.StatusConflict) @@ -601,7 +605,7 @@ func TestRedirectDefinitiveErrorSkipsFallback(t *testing.T) { var entryCalls int entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { entryCalls++ - _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(bad.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{addrOf(bad)}}) })) t.Cleanup(entry.Close) @@ -616,48 +620,11 @@ func TestRedirectDefinitiveErrorSkipsFallback(t *testing.T) { } } -func TestRedirectRotating401CandidateFallsBackToOrigin(t *testing.T) { - var staleCalls int - stale := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - staleCalls++ - w.WriteHeader(http.StatusUnauthorized) - _ = json.MarshalWrite(w, errorResponse{Error: "invalid api token"}) - })) - t.Cleanup(stale.Close) - - var entryCalls int - entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - entryCalls++ - if entryCalls == 1 { - _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(stale.URL, "http://")}}) - return - } - var req claimRequest - _ = json.UnmarshalRead(r.Body, &req) - if !req.NoRedirect { - t.Error("origin fallback did not carry no_redirect") - } - _ = json.MarshalWrite(w, claimResponse{ID: "sb_local", Token: "t"}) - })) - t.Cleanup(entry.Close) - - sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") - if err != nil { - t.Fatalf("New: %v", err) - } - if sb.ID != "sb_local" { - t.Errorf("id %q, want sb_local (provisioned at the already-authorized origin)", sb.ID) - } - if staleCalls != 1 || entryCalls != 2 { - t.Errorf("stale=%d entry=%d calls, want 1 and 2 (redirect + fallback)", staleCalls, entryCalls) - } -} - func TestLookupScatter(t *testing.T) { var owner *httptest.Server owner = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/sandboxes/sb_1/owner" && r.Header.Get("Authorization") == "Bearer tok" { - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(owner.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(owner)}) return } w.WriteHeader(http.StatusNotFound) @@ -669,7 +636,7 @@ func TestLookupScatter(t *testing.T) { case "/v1/sandboxes/sb_1/owner": w.WriteHeader(http.StatusNotFound) case "/v1/peers": - _ = json.MarshalWrite(w, map[string]any{"peers": []string{strings.TrimPrefix(owner.URL, "http://")}}) + _ = json.MarshalWrite(w, map[string]any{"peers": []string{addrOf(owner)}}) } })) t.Cleanup(entry.Close) @@ -678,7 +645,7 @@ func TestLookupScatter(t *testing.T) { if err != nil { t.Fatalf("Lookup: %v", err) } - if sb.ID != "sb_1" || sb.owner != strings.TrimPrefix(owner.URL, "http://") { + if sb.ID != "sb_1" || sb.owner != addrOf(owner) { t.Errorf("handle %+v, want owner=B", sb) } } @@ -732,7 +699,7 @@ func movedSandbox(t *testing.T) (*Sandbox, string) { fake.ServeConn(conn) }) mux.HandleFunc("GET /v1/sandboxes/sb_1/owner", func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(adopter.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(adopter)}) }) mux.HandleFunc("POST /v1/sandboxes/sb_1/renew", func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, map[string]time.Time{"deadline": time.Now().Add(time.Minute)}) @@ -741,14 +708,14 @@ func movedSandbox(t *testing.T) (*Sandbox, string) { t.Cleanup(adopter.Close) old := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/peers" { - _ = json.MarshalWrite(w, map[string]any{"peers": []string{strings.TrimPrefix(adopter.URL, "http://")}}) + _ = json.MarshalWrite(w, map[string]any{"peers": []string{addrOf(adopter)}}) return } w.WriteHeader(http.StatusNotFound) })) t.Cleanup(old.Close) sb := testSandbox(t, old) - return sb, strings.TrimPrefix(adopter.URL, "http://") + return sb, addrOf(adopter) } func retiredAddr(t *testing.T) string { @@ -762,9 +729,37 @@ func retiredAddr(t *testing.T) string { return addr } +func serveFunc(t testing.TB, h http.HandlerFunc) *httptest.Server { + t.Helper() + ts := httptest.NewServer(h) + t.Cleanup(ts.Close) + return ts +} + +func statusServer(t testing.TB, code int, hit *bool) *httptest.Server { + t.Helper() + return serveFunc(t, func(w http.ResponseWriter, _ *http.Request) { + if hit != nil { + *hit = true + } + w.WriteHeader(code) + }) +} + +func redirectEntry(t testing.TB, candidates ...string) *httptest.Server { + t.Helper() + return serveFunc(t, func(w http.ResponseWriter, _ *http.Request) { + _ = json.MarshalWrite(w, claimResponse{Redirect: candidates}) + }) +} + +func addrOf(ts *httptest.Server) string { + return strings.TrimPrefix(ts.URL, "http://") +} + func testClient(t testing.TB, ts *httptest.Server, opts ...ClientOption) *Client { t.Helper() - c, err := Connect(strings.TrimPrefix(ts.URL, "http://"), opts...) + c, err := Connect(addrOf(ts), opts...) if err != nil { t.Fatalf("connect: %v", err) } diff --git a/sdk/go/drain.go b/sdk/go/drain.go index 20d382696..fa4fa82e5 100644 --- a/sdk/go/drain.go +++ b/sdk/go/drain.go @@ -5,8 +5,7 @@ import ( "net/http" ) -// Drain closes the entry node (root token): new claims/forks/branches are -// refused, live claims run to their leases; poll Info until Claimed is zero. +// Drain closes the entry node (root token): new claims/forks/branches are refused, live claims run to their leases; poll Info until Claimed is zero. func (c *Client) Drain(ctx context.Context) (*NodeInfo, error) { return c.doJSONPtr[NodeInfo](ctx, http.MethodPost, c.addr, "/v1/drain", nil, c.apiToken, "drain") } diff --git a/sdk/go/files.go b/sdk/go/files.go index bd98ac4d4..16f4aea01 100644 --- a/sdk/go/files.go +++ b/sdk/go/files.go @@ -10,7 +10,9 @@ import ( // WriteFile atomically writes data to path in the sandbox; a non-nil mode sets the file's permission bits. func (s *Sandbox) WriteFile(ctx context.Context, path string, data []byte, mode *uint32) error { - return s.uploadRPC(ctx, &wire.FsWrite{Path: path, Mode: mode}, bytes.NewReader(data)) + return s.retryMoved(ctx, func() error { + return s.uploadRPC(ctx, &wire.FsWrite{Path: path, Mode: mode}, bytes.NewReader(data)) + }) } // ReadFile returns the contents of path. diff --git a/sdk/go/files_test.go b/sdk/go/files_test.go index eb9dc7040..85da86c2c 100644 --- a/sdk/go/files_test.go +++ b/sdk/go/files_test.go @@ -98,13 +98,6 @@ func TestReadFileToStopsWhenTheWriterRefuses(t *testing.T) { } } -func TestReadMissingFileErrors(t *testing.T) { - sb := fakeSandbox(t) - if _, err := sb.ReadFile(t.Context(), "/nope"); err == nil { - t.Error("read of missing file succeeded") - } -} - func TestSessionLifecycle(t *testing.T) { sb := fakeSandbox(t) ctx := t.Context() diff --git a/sdk/go/git_test.go b/sdk/go/git_test.go index 37f16f720..b67090eb7 100644 --- a/sdk/go/git_test.go +++ b/sdk/go/git_test.go @@ -1,8 +1,11 @@ package sandbox import ( + "net" "slices" "testing" + + "github.com/cocoonstack/sandbox/protocol/wire" ) func TestGitBranchLifecycle(t *testing.T) { @@ -39,11 +42,32 @@ func TestGitBranchLifecycle(t *testing.T) { } func TestGitPushPullPlumbing(t *testing.T) { - sb := fakeSandbox(t) + reqs := make(chan wire.Request, 2) + sb := testSandbox(t, newAgentServer(t, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + sc := wire.NewFrameScanner(conn) + for sc.Scan() { + req, err := wire.DecodeRequest(sc.Bytes()) + if err != nil { + return + } + reqs <- req + line, _ := wire.EncodeResponse(wire.Done{}) + if _, err = conn.Write(append(line, '\n')); err != nil { + return + } + } + })) if err := sb.GitPush(t.Context(), "/repo", ""); err != nil { t.Errorf("push: %v", err) } if err := sb.GitPull(t.Context(), "/repo", "tok"); err != nil { t.Errorf("pull: %v", err) } + if push, ok := (<-reqs).(*wire.GitPush); !ok || push.Path != "/repo" || push.Auth != "" { + t.Errorf("guest saw %+v, want a git push of /repo", push) + } + if pull, ok := (<-reqs).(*wire.GitPull); !ok || pull.Path != "/repo" || pull.Auth != "tok" { + t.Errorf("guest saw %+v, want a git pull of /repo with the token", pull) + } } diff --git a/sdk/go/keepalive_test.go b/sdk/go/keepalive_test.go index ef89a96a7..85f001986 100644 --- a/sdk/go/keepalive_test.go +++ b/sdk/go/keepalive_test.go @@ -14,12 +14,7 @@ import ( ) func TestCallsShareOneConnection(t *testing.T) { - var upgrades atomic.Int32 - fake := silkdtest.NewFake(t.TempDir()) - sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { - upgrades.Add(1) - fake.ServeConn(c) - })) + sb, upgrades := countingSandbox(t, silkdtest.NewFake(t.TempDir()).ServeConn) for range 3 { if _, err := sb.Stat(t.Context(), "/"); err != nil { t.Fatalf("stat: %v", err) @@ -40,12 +35,7 @@ func TestCallsShareOneConnection(t *testing.T) { } func TestRefusedStreamOpenKeepsItsConnection(t *testing.T) { - var upgrades atomic.Int32 - fake := silkdtest.NewFake(t.TempDir()) - sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { - upgrades.Add(1) - fake.ServeConn(c) - })) + sb, upgrades := countingSandbox(t, silkdtest.NewFake(t.TempDir()).ServeConn) for range 3 { if _, err := sb.Watch(t.Context(), "/missing", false); err == nil { t.Fatal("watch on a missing path succeeded") @@ -72,11 +62,7 @@ func TestCanceledCallSendsNothingOnAParkedConnection(t *testing.T) { } func TestKeepAliveOffDialsPerCall(t *testing.T) { - var upgrades atomic.Int32 - sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { - upgrades.Add(1) - silkdtest.ServeConn(c) - }), WithKeepAlive(0)) + sb, upgrades := countingSandbox(t, silkdtest.ServeConn, WithKeepAlive(0)) for range 3 { if _, err := sb.Exec(t.Context(), "echo", "42"); err != nil { t.Fatalf("exec: %v", err) @@ -105,11 +91,7 @@ func TestIdleConnectionCloses(t *testing.T) { } func TestStalledStdinKeepsItsConnectionOut(t *testing.T) { - var upgrades atomic.Int32 - sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { - upgrades.Add(1) - silkdtest.ServeConn(c) - })) + sb, upgrades := countingSandbox(t, silkdtest.ServeConn) pr, pw := io.Pipe() t.Cleanup(func() { _ = pw.Close() }) if code, err := sb.Run(t.Context(), Cmd{Argv: []string{"echo", "hi"}, Stdin: pr}); err != nil || code != 0 { @@ -127,14 +109,12 @@ func TestPeerHangUpIsNoticedBeforeReuse(t *testing.T) { if !canProbe { t.Skip("no parked-connection probe on this platform") } - var upgrades atomic.Int32 gone := make(chan struct{}, 2) fake := silkdtest.NewFake(t.TempDir()) - sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { - upgrades.Add(1) + sb, upgrades := countingSandbox(t, func(c net.Conn) { fake.ServeConn(&hangUpAfterReply{Conn: c}) gone <- struct{}{} - })) + }) if _, err := sb.Stat(t.Context(), "/"); err != nil { t.Fatalf("stat: %v", err) } @@ -184,6 +164,16 @@ func TestPeerQuiet(t *testing.T) { waitUntil(t, func() bool { return !probe.quiet() }, "hang-up not noticed") } +func countingSandbox(t *testing.T, serve func(net.Conn), opts ...ClientOption) (*Sandbox, *atomic.Int32) { + t.Helper() + var upgrades atomic.Int32 + sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { + upgrades.Add(1) + serve(c) + }), opts...) + return sb, &upgrades +} + func waitUntil(t *testing.T, cond func() bool, msg string) { t.Helper() for start := time.Now(); time.Since(start) < 3*time.Second; time.Sleep(time.Millisecond) { diff --git a/sdk/go/options.go b/sdk/go/options.go index e30475499..46c4dce18 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -109,7 +109,6 @@ func WithArchiveOnExpire() Option { return func(r *claimRequest) { r.OnExpire = "archive" } } -// ttlSeconds rounds a lease up to whole wire seconds. func ttlSeconds(d time.Duration) int { return int((d + time.Second - 1) / time.Second) } diff --git a/sdk/go/pools.go b/sdk/go/pools.go index 396a1e9c9..f0c75cbc2 100644 --- a/sdk/go/pools.go +++ b/sdk/go/pools.go @@ -6,7 +6,7 @@ import ( "net/http" ) -// PoolSpec is a desired warm pool for SetPools; egress, warmup, capture_trim, storage, reader, replicate_seconds, egress_upstream_env and tun are config-owned, so it has no field for them. +// PoolSpec is a desired warm pool for SetPools; config-owned pool fields are not settable here. type PoolSpec struct { Template string `json:"template"` Net NetShape `json:"net,omitempty"` diff --git a/sdk/go/pools_test.go b/sdk/go/pools_test.go index 1046da775..93beb763d 100644 --- a/sdk/go/pools_test.go +++ b/sdk/go/pools_test.go @@ -4,7 +4,6 @@ import ( "encoding/json/v2" "net/http" "net/http/httptest" - "strings" "sync" "testing" ) @@ -49,7 +48,7 @@ func TestSetPoolsClusterFansOut(t *testing.T) { } peer := httptest.NewServer(http.HandlerFunc(pools)) defer peer.Close() - peerAddr := strings.TrimPrefix(peer.URL, "http://") + peerAddr := addrOf(peer) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/peers" { diff --git a/sdk/go/port.go b/sdk/go/port.go index 35f3cb0e1..4365d6468 100644 --- a/sdk/go/port.go +++ b/sdk/go/port.go @@ -110,20 +110,23 @@ func (s *Sandbox) ProxyPort(ctx context.Context, localAddr string, port uint16) return l, nil } -// openStream turns a call into a raw byte stream, shared by DialPort and Lsp.Request. -func (s *Sandbox) openStream(ctx context.Context, req wire.Request) (_ *PortConn, err error) { - defer func() { s.rebind(ctx, err) }() - conn, l, err := s.call(ctx, req) - if err != nil { - return nil, err - } - if _, err = expect[wire.Ready](ctx, conn); err != nil { - return nil, l.done(err) - } - pr, pw := io.Pipe() - p := &PortConn{conn: conn, stop: l.close, out: pr} - go p.drain(ctx, pw) - return p, nil +// openStream turns a call into a raw byte stream. +func (s *Sandbox) openStream(ctx context.Context, req wire.Request) (*PortConn, error) { + var p *PortConn + err := s.retryMoved(ctx, func() error { + conn, l, err := s.call(ctx, req) + if err != nil { + return err + } + if _, err = expect[wire.Ready](ctx, conn); err != nil { + return l.done(err) + } + pr, pw := io.Pipe() + p = &PortConn{conn: conn, stop: l.close, out: pr} + go p.drain(ctx, pw) + return nil + }) + return p, err } func (s *Sandbox) proxyConn(ctx context.Context, local net.Conn, port uint16) { diff --git a/sdk/go/probe_unix.go b/sdk/go/probe_unix.go index f38925190..cabde0f7b 100644 --- a/sdk/go/probe_unix.go +++ b/sdk/go/probe_unix.go @@ -2,7 +2,10 @@ package sandbox -import "syscall" +import ( + "errors" + "syscall" +) const canProbe = true @@ -10,6 +13,6 @@ const canProbe = true func (c *agentConn) peek(fd uintptr) bool { var b [1]byte _, _, err := syscall.Recvfrom(int(fd), b[:], syscall.MSG_PEEK) - c.peeked = err == syscall.EAGAIN + c.peeked = errors.Is(err, syscall.EAGAIN) return true } diff --git a/sdk/go/proc.go b/sdk/go/proc.go index 17b4c7a02..7675eed00 100644 --- a/sdk/go/proc.go +++ b/sdk/go/proc.go @@ -52,17 +52,21 @@ func (s *Sandbox) Attach(ctx context.Context, pid uint32, stdout, stderr io.Writ return s.drainProc(ctx, &wire.Attach{PID: pid}, stdout, stderr) } -func (s *Sandbox) drainProc(ctx context.Context, req wire.Request, stdout, stderr io.Writer) (_ int32, _ bool, err error) { - defer func() { s.rebind(ctx, err) }() - conn, l, err := s.call(ctx, req) - if err != nil { - return 0, false, err - } - defer l.close() - _, code, exited, err := pumpStdio(ctx, conn, stdout, stderr) - // Logs closes with done after the exit frame - if _, replay := req.(*wire.Logs); replay && exited && err == nil { - err = terminalErr(ctx, conn) - } - return code, exited, l.done(err) +func (s *Sandbox) drainProc(ctx context.Context, req wire.Request, stdout, stderr io.Writer) (int32, bool, error) { + var code int32 + var exited bool + err := s.retryMoved(ctx, func() error { + conn, l, err := s.call(ctx, req) + if err != nil { + return err + } + defer l.close() + _, code, exited, err = pumpStdio(ctx, conn, stdout, stderr) + // Logs closes with done after the exit frame + if _, replay := req.(*wire.Logs); replay && exited && err == nil { + err = terminalErr(ctx, conn) + } + return l.done(err) + }) + return code, exited, err } diff --git a/sdk/go/pty.go b/sdk/go/pty.go index f037bd276..93b82db51 100644 --- a/sdk/go/pty.go +++ b/sdk/go/pty.go @@ -87,19 +87,21 @@ func (p *Pty) drain(ctx context.Context, pw *io.PipeWriter, stop func()) { } // OpenPty starts a shell whose lifetime is governed by ctx or Close. -func (s *Sandbox) OpenPty(ctx context.Context, opts wire.PtyOpen) (_ *Pty, err error) { - defer func() { s.rebind(ctx, err) }() - conn, l, err := s.call(ctx, &opts) - if err != nil { - return nil, err - } - started, err := expect[wire.Started](ctx, conn) - if err != nil { - return nil, l.done(err) - } - - pr, pw := io.Pipe() - p := &Pty{PID: started.PID, sb: s, conn: conn, stop: l.close, out: pr} - go p.drain(ctx, pw, l.close) - return p, nil +func (s *Sandbox) OpenPty(ctx context.Context, opts wire.PtyOpen) (*Pty, error) { + var p *Pty + err := s.retryMoved(ctx, func() error { + conn, l, err := s.call(ctx, &opts) + if err != nil { + return err + } + started, err := expect[wire.Started](ctx, conn) + if err != nil { + return l.done(err) + } + pr, pw := io.Pipe() + p = &Pty{PID: started.PID, sb: s, conn: conn, stop: l.close, out: pr} + go p.drain(ctx, pw, l.close) + return nil + }) + return p, err } diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index 48bae30db..7b85e326c 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -65,8 +65,39 @@ func (e *ownerGoneError) Error() string { return e.err.Error() } func (e *ownerGoneError) Unwrap() error { return e.err } -// Sandbox is a claimed sandbox handle; all data-plane calls dial its owning -// node directly. +type forkRequest struct { + Token string `json:"token"` + Count int `json:"count"` + TTLSeconds int `json:"ttl_seconds,omitzero"` +} + +type forkResponse struct { + Children []claimResponse `json:"children"` +} + +type promoteRequest struct { + Token string `json:"token"` + Template string `json:"template"` +} + +type promoteResponse struct { + Key struct { + Template string `json:"template"` + Net string `json:"net"` + Size string `json:"size"` + } `json:"key"` + ContentDigest string `json:"content_digest"` +} + +type renewRequest struct { + TTLSeconds int `json:"ttl_seconds,omitzero"` +} + +type renewResponse struct { + Deadline time.Time `json:"deadline"` +} + +// Sandbox is a claimed sandbox handle; all data-plane calls dial its owning node directly. type Sandbox struct { ID string Deadline time.Time @@ -76,10 +107,8 @@ type Sandbox struct { VolumesWant []Volume // TemplateDigest is the content identity of the promoted-template export this sandbox was cloned from; empty otherwise. TemplateDigest string - // FromCheckpoint names the checkpoint this sandbox branched from; empty otherwise. FromCheckpoint string - // NetRoute is the guest's network path at claim, as the node reports it. NetRoute string @@ -91,22 +120,19 @@ type Sandbox struct { owner string } -// Owner returns the data-plane address of the node that owns the sandbox — -// on a cluster the node a redirected claim landed on. +// Owner returns the data-plane address of the node that owns the sandbox, on a cluster the node a redirected claim landed on. func (s *Sandbox) Owner() string { s.mu.Lock() defer s.mu.Unlock() return s.owner } -// Token is the per-sandbox bearer secret — persist it with ID to reattach -// later via Lookup (treat it like a credential). +// Token is the per-sandbox bearer secret; persist it with ID to reattach later via Lookup, and treat it like a credential. func (s *Sandbox) Token() string { return s.token } -// Exec runs argv to completion and returns its stdout; a non-zero exit -// surfaces as *ExitError carrying stderr, alongside the partial stdout. +// Exec runs argv to completion and returns its stdout; a non-zero exit surfaces as *ExitError carrying stderr, alongside the partial stdout. func (s *Sandbox) Exec(ctx context.Context, argv ...string) (string, error) { var stdout, stderr strings.Builder code, err := s.Run(ctx, Cmd{Argv: argv, Stdout: &stdout, Stderr: &stderr}) @@ -119,8 +145,7 @@ func (s *Sandbox) Exec(ctx context.Context, argv ...string) (string, error) { return stdout.String(), nil } -// Run executes cmd in the sandbox, streaming stdio over one relayed silkd -// connection, and returns the exit code; a ctx that ends first kills the command. +// Run executes cmd in the sandbox, streaming stdio over one relayed silkd connection, and returns the exit code; a ctx that ends first kills the command. func (s *Sandbox) Run(ctx context.Context, cmd Cmd) (int, error) { if len(cmd.Argv) == 0 { return 0, fmt.Errorf("empty argv") @@ -185,8 +210,7 @@ func (s *Sandbox) Renew(ctx context.Context, ttl time.Duration) (time.Time, erro return rr.Deadline, nil } -// Close releases the sandbox on its node; releasing one already gone is not -// an error. It takes no ctx so it stays defer-friendly — bounded internally. +// Close releases the sandbox on its node, bounded internally and without a ctx to stay defer-friendly; releasing one already gone is not an error. func (s *Sandbox) Close() error { s.pool.drain() ctx, cancel := context.WithTimeout(context.Background(), releaseTimeout) @@ -211,6 +235,7 @@ func (s *Sandbox) run(ctx context.Context, cmd Cmd) (int, error) { defer l.close() stdinDone := make(chan struct{}) + var stdinErr error if cmd.Stdin == nil { // A guest that already answered and closed fails this send; the frames it sent still come back below _ = conn.Send(wire.StdinClose{}) @@ -218,15 +243,16 @@ func (s *Sandbox) run(ctx context.Context, cmd Cmd) (int, error) { } else { go func() { defer close(stdinDone) - pumpStdin(conn, cmd.Stdin) + stdinErr = pumpStdin(conn, cmd.Stdin) }() } pid, code, exited, err := pumpStdio(ctx, conn, cmd.Stdout, cmd.Stderr) // A pump still parked in Read would write into the next RPC, so its connection is not kept + pumped := false select { case <-stdinDone: - err = l.done(err) + err, pumped = l.done(err), true default: } if err != nil { @@ -240,6 +266,9 @@ func (s *Sandbox) run(ctx context.Context, cmd Cmd) (int, error) { if !exited { return 0, unexpected(&wire.Done{}) } + if pumped && stdinErr != nil { + return int(code), &callerError{fmt.Errorf("read stdin: %w", stdinErr)} + } return int(code), nil } @@ -354,18 +383,21 @@ func markGone(err error, id, addr string) error { } // pumpStdin chunks the reader into stdin frames. -func pumpStdin(conn *silkd.Conn, r io.Reader) { +func pumpStdin(conn *silkd.Conn, r io.Reader) error { buf := make([]byte, stdinChunk) for { n, err := r.Read(buf) if n > 0 { if sendErr := conn.Send(&wire.Stdin{Data: buf[:n]}); sendErr != nil { - return + return nil } } if err != nil { _ = conn.Send(wire.StdinClose{}) - return + if errors.Is(err, io.EOF) { + return nil + } + return err } } } diff --git a/sdk/go/sandbox_test.go b/sdk/go/sandbox_test.go index f29455d82..c8f0f09ca 100644 --- a/sdk/go/sandbox_test.go +++ b/sdk/go/sandbox_test.go @@ -12,6 +12,7 @@ import ( "slices" "strings" "testing" + "testing/iotest" "time" "github.com/cocoonstack/sandbox/protocol/wire" @@ -69,6 +70,21 @@ func TestRunStreamsStdin(t *testing.T) { } } +func TestRunReportsAStdinReadFailure(t *testing.T) { + sb := testSandbox(t, newAgentServer(t, silkdtest.ServeConn)) + broken := errors.New("upstream reset") + + var out strings.Builder + code, err := sb.Run(t.Context(), Cmd{ + Argv: []string{"cat"}, + Stdin: io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(broken)), + Stdout: &out, + }) + if !errors.Is(err, broken) || code != 0 || out.String() != "partial" { + t.Errorf("code=%d out=%q err=%v, want the exit code with the stdin failure", code, out.String(), err) + } +} + func TestRunReturnsAtItsContextDeadline(t *testing.T) { sb := testSandbox(t, newAgentServer(t, silkdtest.ServeConn)) diff --git a/sdk/go/session.go b/sdk/go/session.go index 71a29e452..332c89cd7 100644 --- a/sdk/go/session.go +++ b/sdk/go/session.go @@ -7,16 +7,14 @@ import ( "github.com/cocoonstack/sandbox/protocol/wire" ) -// Session is a persistent shell in the sandbox: cwd, env, and shell state -// survive across Exec calls until Close. +// Session is a persistent shell in the sandbox: cwd, env, and shell state survive across Exec calls until Close. type Session struct { ID string sb *Sandbox } -// Exec runs argv in the session and returns its combined output; state (cwd, -// env, shell variables) persists to the next call. +// Exec runs argv in the session and returns its combined output; state (cwd, env, shell variables) persists to the next call. func (sess *Session) Exec(ctx context.Context, argv ...string) (string, error) { var out strings.Builder code, err := sess.sb.Run(ctx, Cmd{Argv: argv, Session: sess.ID, Stdout: &out, Stderr: &out}) diff --git a/sdk/go/silkd/conn.go b/sdk/go/silkd/conn.go index 6d6931683..18d74c107 100644 --- a/sdk/go/silkd/conn.go +++ b/sdk/go/silkd/conn.go @@ -11,9 +11,7 @@ import ( "github.com/cocoonstack/sandbox/protocol/wire" ) -// Conn frames a byte stream (typically the upgraded relay connection) into -// silkd RPCs. Reads are single-consumer; Send is safe for concurrent use so -// a stdin pump can interleave with the caller. +// Conn frames a byte stream (typically the upgraded relay connection) into silkd RPCs; reads are single-consumer, and Send is safe for concurrent use so a stdin pump can interleave. type Conn struct { wmu sync.Mutex wbuf []byte diff --git a/sdk/go/silkd/silkdtest/fake.go b/sdk/go/silkd/silkdtest/fake.go index 4c699a604..2d0fff1d6 100644 --- a/sdk/go/silkd/silkdtest/fake.go +++ b/sdk/go/silkd/silkdtest/fake.go @@ -36,8 +36,7 @@ func NewFake(root string) *Fake { return &Fake{Root: root, sessions: map[string]bool{}, branches: []string{"main"}, current: "main"} } -// ServeConn serves RPCs on an already-open connection (e.g. after an HTTP -// hijack) until the peer closes it. +// ServeConn serves RPCs on an already-open connection (e.g. after an HTTP hijack) until the peer closes it. func (f *Fake) ServeConn(conn net.Conn) { serveConn(conn, bufio.NewReader(conn), f.serve) } @@ -118,9 +117,9 @@ func (f *Fake) fsList(conn net.Conn, path string) { var entries []wire.DirEntry for _, e := range ents { info, _ := e.Info() - kind := "file" + kind := wire.FileKindFile if e.IsDir() { - kind = "dir" + kind = wire.FileKindDir } var size uint64 if info != nil { @@ -138,9 +137,9 @@ func (f *Fake) fsStat(conn net.Conn, path string) { errFrame(conn, wire.KindNotFound, err.Error()) return } - kind := "file" + kind := wire.FileKindFile if info.IsDir() { - kind = "dir" + kind = wire.FileKindDir } send(conn, &wire.Stat{Info: wire.FileInfo{ Kind: kind, @@ -225,8 +224,7 @@ func (f *Fake) fsPull(conn net.Conn, path string) { return } _ = tw.Close() - data := buf.Bytes() - sendChunked(conn, data) + sendChunked(conn, buf.Bytes()) } func (f *Fake) sessionCreate(conn net.Conn, req *wire.SessionCreate) { @@ -337,7 +335,7 @@ func (f *Fake) fsWatch(conn net.Conn, r *bufio.Reader, req *wire.FsWatch) { return } send(conn, wire.Ready{}) - send(conn, &wire.Event{Kind: "created", Path: req.Path + "/silkdtest-event"}) + send(conn, &wire.Event{Kind: wire.EventCreated, Path: req.Path + "/silkdtest-event"}) _, _ = io.Copy(io.Discard, r) } diff --git a/sdk/go/template.go b/sdk/go/template.go index 30fb7aafb..12d855bce 100644 --- a/sdk/go/template.go +++ b/sdk/go/template.go @@ -1,9 +1,6 @@ package sandbox -import ( - "context" - "net/url" -) +import "context" // Template is a promoted template bound to the node that holds it. type Template struct { @@ -18,10 +15,7 @@ type Template struct { // New claims a sandbox cloned from the template; its key axes cannot be overridden. func (t *Template) New(ctx context.Context, opts ...Option) (*Sandbox, error) { - claim := claimRequest{Template: t.Name} - for _, opt := range opts { - opt(&claim) - } + claim := newClaim(t.Name, opts) if err := claim.rejectPinnedAxes(); err != nil { return nil, err } @@ -41,12 +35,8 @@ func (t *Template) New(ctx context.Context, opts ...Option) (*Sandbox, error) { // Delete removes the template from its owning node, never a same-name template elsewhere. func (t *Template) Delete(ctx context.Context) error { - u := url.Values{ - templateQueryParam: {t.Name}, - netQueryParam: {t.net}, - sizeQueryParam: {t.size}, - noRedirectQueryParam: {"1"}, - } + u := templateQuery(t.Name, t.net, t.size) + u.Set(noRedirectQueryParam, "1") _, err := t.c.deleteTemplates(ctx, t.addr, u) return err } diff --git a/sdk/go/tenants_test.go b/sdk/go/tenants_test.go index d55bedf0f..f37cab5e2 100644 --- a/sdk/go/tenants_test.go +++ b/sdk/go/tenants_test.go @@ -7,7 +7,6 @@ import ( "maps" "net/http" "net/http/httptest" - "strings" "sync" "testing" ) @@ -81,7 +80,7 @@ func TestDeleteTenantClusterCountsAMissingTenantAsDone(t *testing.T) { defer broken.Close() old := httptest.NewServer(http.NotFoundHandler()) defer old.Close() - peers := []string{strings.TrimPrefix(gone.URL, "http://"), strings.TrimPrefix(broken.URL, "http://"), strings.TrimPrefix(old.URL, "http://")} + peers := []string{addrOf(gone), addrOf(broken), addrOf(old)} entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/peers" { _ = json.MarshalWrite(w, map[string][]string{"peers": peers}) diff --git a/sdk/go/tree.go b/sdk/go/tree.go index a0efd448f..1f0376a43 100644 --- a/sdk/go/tree.go +++ b/sdk/go/tree.go @@ -7,9 +7,7 @@ import ( "github.com/cocoonstack/sandbox/protocol/wire" ) -// Push extracts a tar stream under dest in the sandbox — the no-network lane's -// way to move a project in. tarStream is a reader of tar bytes (e.g. from -// archive/tar); silkd runs `tar -x` under dest. +// Push extracts a tar stream under dest in the sandbox, the no-network lane's way to move a project in. func (s *Sandbox) Push(ctx context.Context, dest string, tarStream io.Reader) error { return s.uploadRPC(ctx, &wire.FsPush{Dest: dest}, tarStream) } diff --git a/sdk/go/volumes_test.go b/sdk/go/volumes_test.go index e2d46cb75..efd554d9e 100644 --- a/sdk/go/volumes_test.go +++ b/sdk/go/volumes_test.go @@ -94,7 +94,7 @@ func TestTemplateNewVolumeClaimFollowsRedirect(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { originAuth = r.Header.Get("Authorization") _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(target.URL, "http://")}, RequirePromoted: true, + Redirect: []string{addrOf(target)}, RequirePromoted: true, }) })) t.Cleanup(entry.Close) @@ -162,7 +162,7 @@ func TestVolumeClaimRedirectPreservesEntries(t *testing.T) { t.Cleanup(target.Close) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(target.URL, "http://")}, + Redirect: []string{addrOf(target)}, }) })) t.Cleanup(entry.Close) diff --git a/sdk/go/watch.go b/sdk/go/watch.go index 4f5495ffe..61ce0984f 100644 --- a/sdk/go/watch.go +++ b/sdk/go/watch.go @@ -76,20 +76,23 @@ func (w *Watcher) setErr(err error) { } // Watch streams filesystem events under path; events caused after it returns are guaranteed captured. -func (s *Sandbox) Watch(ctx context.Context, path string, recursive bool) (_ *Watcher, err error) { - defer func() { s.rebind(ctx, err) }() - conn, l, err := s.call(ctx, &wire.FsWatch{Path: path, Recursive: recursive}) - if err != nil { - return nil, err - } - if _, err = expect[wire.Ready](ctx, conn); err != nil { - return nil, l.done(err) - } - w := &Watcher{ - events: make(chan wire.Event, 16), - stop: l.close, - closed: make(chan struct{}), - } - go w.drain(ctx, conn) - return w, nil +func (s *Sandbox) Watch(ctx context.Context, path string, recursive bool) (*Watcher, error) { + var w *Watcher + err := s.retryMoved(ctx, func() error { + conn, l, err := s.call(ctx, &wire.FsWatch{Path: path, Recursive: recursive}) + if err != nil { + return err + } + if _, err = expect[wire.Ready](ctx, conn); err != nil { + return l.done(err) + } + w = &Watcher{ + events: make(chan wire.Event, 16), + stop: l.close, + closed: make(chan struct{}), + } + go w.drain(ctx, conn) + return nil + }) + return w, err } diff --git a/sdk/langchain/cocoonsandbox_langchain/toolkit.py b/sdk/langchain/cocoonsandbox_langchain/toolkit.py index f4fd34a94..b4527bb48 100644 --- a/sdk/langchain/cocoonsandbox_langchain/toolkit.py +++ b/sdk/langchain/cocoonsandbox_langchain/toolkit.py @@ -12,7 +12,6 @@ from langchain_core.tools import StructuredTool, ToolException from pydantic import BaseModel, Field -# one call's wall clock over the claim, the dial and the command; the tool description states it CALL_TIMEOUT = 300 @@ -70,7 +69,7 @@ def get_tools(self) -> list[StructuredTool]: "Returns stdout; a non-empty stderr is appended as a 'stderr:' " "line and a non-zero status as an 'exit code: N' line; a " "command that prints nothing and exits 0 returns '(no output)'. " - "The call is cut off after 5 minutes and the reply says so. " + f"The call is cut off after {CALL_TIMEOUT // 60} minutes and the reply says so. " "Files and installed packages persist across calls; environment " "variables and the working directory do not.", ExecInput, diff --git a/sdk/langchain/tests/test_toolkit.py b/sdk/langchain/tests/test_toolkit.py index f792230d2..4e882fcfd 100644 --- a/sdk/langchain/tests/test_toolkit.py +++ b/sdk/langchain/tests/test_toolkit.py @@ -109,15 +109,22 @@ def claim(deadline=None): assert kit.get_tools()[0].invoke({"command": "echo hi"}) == f"cut off after {CALL_TIMEOUT}s" -def test_a_guest_error_comes_back_as_a_tool_error(monkeypatch): +@pytest.mark.parametrize( + ("name", "method", "args", "message"), + [ + ("sandbox_write_file", "write_file", {"path": "/nope/a.txt", "content": "body"}, "no such directory"), + ("sandbox_read_file", "read_file", {"path": "/nope"}, "no such file"), + ], +) +def test_a_guest_error_comes_back_as_a_tool_error(monkeypatch, name, method, args, message): kit, fake = hooked(monkeypatch) - def refused(path, data): - raise SilkdError("not_found", "no such directory") + def refused(*_): + raise SilkdError("not_found", message) - fake.write_file = refused - tool = next(t for t in kit.get_tools() if t.name == "sandbox_write_file") - assert "no such directory" in tool.invoke({"path": "/nope/a.txt", "content": "body"}) + setattr(fake, method, refused) + tool = next(t for t in kit.get_tools() if t.name == name) + assert message in tool.invoke(args) def test_default_lease_outlives_an_agent_run(monkeypatch): @@ -160,17 +167,6 @@ def test_a_failed_release_never_hides_the_block_error(monkeypatch): kit.get_tools()[0].invoke({"command": "x"}) -def test_read_file_reports_a_missing_path_as_a_tool_error(monkeypatch): - kit, fake = hooked(monkeypatch) - - def missing(path): - raise SilkdError("not_found", "no such file") - - fake.read_file = missing - tool = next(t for t in kit.get_tools() if t.name == "sandbox_read_file") - assert "no such file" in tool.invoke({"path": "/nope"}) - - def test_a_stalled_agent_upgrade_comes_back_as_a_tool_error(): server = socket.create_server(("127.0.0.1", 0)) addr = f"127.0.0.1:{server.getsockname()[1]}" diff --git a/sdk/openai/cocoonsandbox_openai/adapter.py b/sdk/openai/cocoonsandbox_openai/adapter.py index 0a2587e98..ca0ff7bcd 100644 --- a/sdk/openai/cocoonsandbox_openai/adapter.py +++ b/sdk/openai/cocoonsandbox_openai/adapter.py @@ -162,7 +162,9 @@ async def create( with contextlib.suppress(Exception): # the construction error is the one worth reading await asyncio.to_thread(sb.close) raise - return self._wrap_session(CocoonSandboxSession.from_state(state)) + session = CocoonSandboxSession.from_state(state) + session._sb = sb + return self._wrap_session(session) async def delete(self, session: SandboxSession) -> SandboxSession: inner = session._inner diff --git a/sdk/openai/tests/test_adapter.py b/sdk/openai/tests/test_adapter.py index 2939cdeba..62d96a03a 100644 --- a/sdk/openai/tests/test_adapter.py +++ b/sdk/openai/tests/test_adapter.py @@ -92,19 +92,13 @@ async def go(): def test_exec_maps_stdio_and_exit(node, monkeypatch): class FakeSandbox: - def __init__(self, **kw): - self.id = "sb_1" - def run(self, argv, on_stdout=None, on_stderr=None, timeout=None): assert argv == ["echo", "hi"] and timeout is None on_stdout(b"hi\n") return 0 async def go(): - client = CocoonSandboxClient() - session = await client.create(options=CocoonSandboxClientOptions(addr=node)) - inner = session._inner - monkeypatch.setattr(inner, "_sandbox", lambda: FakeSandbox()) + inner = await _session_with(node, monkeypatch, FakeSandbox()) result = await inner._exec_internal("echo", "hi") assert result.exit_code == 0 and result.stdout == b"hi\n" assert result.ok() @@ -124,10 +118,7 @@ def run(self, argv, on_stdout=None, on_stderr=None, timeout=None): return 0 async def go(): - client = CocoonSandboxClient() - session = await client.create(options=CocoonSandboxClientOptions(addr=node)) - inner = session._inner - monkeypatch.setattr(inner, "_sandbox", lambda: FakeSandbox()) + inner = await _session_with(node, monkeypatch, FakeSandbox()) with pytest.raises(TimeoutError): await inner._exec_internal("sleep", "9", timeout=1.5) result = await inner._exec_internal("sleep", "9") @@ -142,10 +133,7 @@ def read_file(self, path): raise SilkdError("not_found", path) async def go(): - client = CocoonSandboxClient() - session = await client.create(options=CocoonSandboxClientOptions(addr=node)) - inner = session._inner - monkeypatch.setattr(inner, "_sandbox", lambda: FakeSandbox()) + inner = await _session_with(node, monkeypatch, FakeSandbox()) with pytest.raises(FileNotFoundError): await inner.read(Path("/nope")) @@ -158,10 +146,7 @@ def stat(self, path): raise ProtocolError("dial 127.0.0.1:1: connection refused") async def go(): - client = CocoonSandboxClient() - session = await client.create(options=CocoonSandboxClientOptions(addr=node)) - inner = session._inner - monkeypatch.setattr(inner, "_sandbox", lambda: FakeSandbox()) + inner = await _session_with(node, monkeypatch, FakeSandbox()) assert await inner.running() is False asyncio.run(go()) @@ -182,10 +167,7 @@ def push(self, dest, data): calls["push"] = (dest, data) async def go(): - client = CocoonSandboxClient() - session = await client.create(options=CocoonSandboxClientOptions(addr=node)) - inner = session._inner - monkeypatch.setattr(inner, "_sandbox", lambda: FakeSandbox()) + inner = await _session_with(node, monkeypatch, FakeSandbox()) await inner.write(Path("/workspace/a.txt"), io.BytesIO(b"body")) assert calls["write"] == ("/workspace/a.txt", b"body") tar = await inner.persist_workspace() @@ -198,15 +180,20 @@ async def go(): asyncio.run(go()) -def test_create_keeps_the_construction_error_when_the_release_also_fails(node, monkeypatch): +@pytest.mark.parametrize("release_fails", [False, True]) +def test_create_releases_the_claim_and_keeps_the_construction_error(node, monkeypatch, release_fails): + released = [] + def bad_snapshot(*args, **kwargs): raise ValueError("bad snapshot") - def bad_close(self): - raise RuntimeError("node gone") + def close(self): + released.append(self.id) + if release_fails: + raise RuntimeError("node gone") monkeypatch.setattr("cocoonsandbox_openai.adapter.resolve_snapshot", bad_snapshot) - monkeypatch.setattr("cocoonsandbox.Sandbox.close", bad_close) + monkeypatch.setattr("cocoonsandbox.Sandbox.close", close) async def go(): client = CocoonSandboxClient() @@ -214,24 +201,14 @@ async def go(): await client.create(options=CocoonSandboxClientOptions(addr=node)) asyncio.run(go()) + assert released == ["sb_1"] -def test_create_releases_the_claim_when_state_construction_fails(node, monkeypatch): - released = [] - - def bad_snapshot(*args, **kwargs): - raise ValueError("bad snapshot") - - monkeypatch.setattr("cocoonsandbox_openai.adapter.resolve_snapshot", bad_snapshot) - monkeypatch.setattr("cocoonsandbox.Sandbox.close", lambda self: released.append(self.id)) - - async def go(): - client = CocoonSandboxClient() - with pytest.raises(ValueError): - await client.create(options=CocoonSandboxClientOptions(addr=node)) - - asyncio.run(go()) - assert released == ["sb_1"] +async def _session_with(node, monkeypatch, fake): + session = await CocoonSandboxClient().create(options=CocoonSandboxClientOptions(addr=node)) + inner = session._inner + monkeypatch.setattr(inner, "_sandbox", lambda: fake) + return inner def _silkd_pull_tar(root: str, files: dict[str, bytes]) -> bytes: diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 0dfebfb8f..0790dc76d 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -5,6 +5,7 @@ import http.client import json import queue +import socket import ssl import threading import time @@ -13,20 +14,20 @@ from typing import Any, TypeVar, cast from .checkpoint import Checkpoint -from .conn import Conn, dial_agent, is_quiet, remaining_timeout +from .conn import Conn, _Closeable, dial_agent, is_quiet, remaining_timeout from .endpoint import _endpoint_url from .errors import APIError, SandboxTimeout, _error_message, _require_field from .sandbox import Sandbox from .template import _claim_body, _template_query from .volumes import Volume -T = TypeVar("T") +_T = TypeVar("_T") _PEERS_TIMEOUT = 5.0 _IDLE_CONNS = 8 -class Client: +class Client(_Closeable): """Talks to one sandboxd node (and, transparently, its cluster).""" def __init__( @@ -42,7 +43,7 @@ def __init__( ssl_context is set to offer ALPN http/1.1, which the relay upgrade needs; pass a dedicated context. """ - endpoint = _endpoint_url(addr.split(",")[0].strip()) + endpoint = _endpoint_url(addr.strip()) self.addr = endpoint.geturl().removeprefix("http://") self._scheme = endpoint.scheme self._ssl_context = ssl_context @@ -145,6 +146,14 @@ def info(self) -> dict[str, Any]: """The node's pool/claim counters, as served by GET /v1/info.""" return self._request(self.addr, "GET", "/v1/info", None, "info") + def close(self) -> None: + """Closes the parked control connections; a later call dials afresh.""" + with self._idle_lock: + while self._idle: + _, idle = self._idle.popitem() + for conn, _ in idle: + conn.close() + def _claim_from( self, addr: str, @@ -288,8 +297,8 @@ def _retry_miss(exc: APIError) -> bool: def _try_each( - candidates: Iterable[str], call: Callable[[str], T], retry: Callable[[APIError], bool] = _retry_miss -) -> T: + candidates: Iterable[str], call: Callable[[str], _T], retry: Callable[[APIError], bool] = _retry_miss +) -> _T: last_error = None for addr in candidates: try: @@ -302,7 +311,13 @@ def _try_each( def _retry_transient(exc: APIError) -> bool: - return exc.status in (0, 401, 404, 429, 503, 500, 502, 504) + return exc.status in (0, 401, 404, 429, 500, 503) + + +def _claim_refused(exc: APIError) -> bool: + if exc.status: + return exc.status not in (502, 504) + return exc.__cause__ is None or isinstance(exc.__cause__, (ConnectionRefusedError, socket.gaierror)) def _redirect_fallback( @@ -315,9 +330,9 @@ def attempt(addr: str) -> tuple[str, dict[str, Any]]: return addr, reply try: - return _try_each(candidates, attempt, retry=lambda _: True) + return _try_each(candidates, attempt, retry=_claim_refused) except APIError as exc: - if not _retry_transient(exc): + if not (_claim_refused(exc) and _retry_transient(exc)): raise try: return attempt(origin) @@ -326,8 +341,8 @@ def attempt(addr: str) -> tuple[str, dict[str, Any]]: raise APIError(verb, origin_exc.status, combined) from origin_exc -def _scatter(addrs: Sequence[str], probe: Callable[[str], T]) -> T: - results: queue.Queue[tuple[T | None, Exception | None]] = queue.Queue(maxsize=len(addrs)) +def _scatter(addrs: Sequence[str], probe: Callable[[str], _T]) -> _T: + results: queue.Queue[tuple[_T | None, Exception | None]] = queue.Queue(maxsize=len(addrs)) def run(addr: str) -> None: try: @@ -341,6 +356,6 @@ def run(addr: str) -> None: for _ in addrs: value, error = results.get() if error is None: - return cast(T, value) + return cast(_T, value) last_error = error raise cast(Exception, last_error) diff --git a/sdk/python/cocoonsandbox/conn.py b/sdk/python/cocoonsandbox/conn.py index e8d7a3ca3..697d8cf2f 100644 --- a/sdk/python/cocoonsandbox/conn.py +++ b/sdk/python/cocoonsandbox/conn.py @@ -15,7 +15,7 @@ from .errors import APIError, ProtocolError, SandboxError, SandboxTimeout, SilkdError, _error_message, _mark_gone from .frames import MAX_FRAME, decode_response, encode_request -KEEP_ALIVE_CONNS = 8 +_KEEP_ALIVE_CONNS = 8 _FD_SETSIZE = 1024 _READ_BUFFER = 64 * 1024 _TCP_KEEPALIVE = {"TCP_KEEPIDLE": 15, "TCP_KEEPALIVE": 15, "TCP_KEEPINTVL": 15, "TCP_KEEPCNT": 9} @@ -146,7 +146,7 @@ def take(self) -> Conn | None: def park(self, conn: Conn) -> None: with self._lock: - if self._idle > 0 and len(self._parked) < KEEP_ALIVE_CONNS: + if self._idle > 0 and len(self._parked) < _KEEP_ALIVE_CONNS: self._parked.append((time.monotonic() + self._idle, conn)) if self._sweep is None: self._arm(self._idle) diff --git a/sdk/python/cocoonsandbox/endpoint.py b/sdk/python/cocoonsandbox/endpoint.py index 42c53e6e5..062e57be5 100644 --- a/sdk/python/cocoonsandbox/endpoint.py +++ b/sdk/python/cocoonsandbox/endpoint.py @@ -5,11 +5,11 @@ import urllib.parse -def _endpoint_url(addr: str, scheme: str = "http") -> urllib.parse.SplitResult: +def _endpoint_url(addr: str) -> urllib.parse.SplitResult: if any(ord(c) <= 32 or ord(c) == 127 for c in addr): raise ValueError("sandboxd endpoint contains whitespace or a control character") if "://" not in addr: - addr = f"{scheme}://{addr}" + addr = f"http://{addr}" endpoint = urllib.parse.urlsplit(addr) if ( endpoint.scheme not in ("http", "https") diff --git a/sdk/python/cocoonsandbox/errors.py b/sdk/python/cocoonsandbox/errors.py index 0112610dd..221bf30ff 100644 --- a/sdk/python/cocoonsandbox/errors.py +++ b/sdk/python/cocoonsandbox/errors.py @@ -78,6 +78,6 @@ def _error_message(raw: bytes) -> str: def _mark_gone(exc: _E, tag: tuple[str, str]) -> _E: - """Tags an owner's 404 or refused dial with its (sandbox id, owner address); callback errors stay untagged.""" + """Tags an owner's 404 or refused dial with its (sandbox id, owner address).""" exc._owner_gone = tag return exc diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index cfe517daf..ee50e49e3 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -566,7 +566,7 @@ def _proxy_accept_loop(self, listener: socket.socket, port: int) -> None: while True: try: local, _ = listener.accept() - except socket.timeout: + except TimeoutError: continue except OSError: return diff --git a/sdk/python/e2e.py b/sdk/python/e2e.py index 53ee44301..cf05c40ec 100644 --- a/sdk/python/e2e.py +++ b/sdk/python/e2e.py @@ -149,7 +149,7 @@ def _promote(): @step("port") def _port(): - deadline = time.time() + 5 + deadline = time.monotonic() + 5 while True: try: conn = sb.dial_port(22) @@ -158,20 +158,20 @@ def _port(): assert banner.startswith(b"SSH-"), banner return except SilkdError: - if time.time() > deadline: + if time.monotonic() > deadline: raise time.sleep(0.3) try: for name, fn in steps: - start = time.time() + start = time.monotonic() try: fn() except BaseException: print(f" {name:<18} FAIL") traceback.print_exc() raise - print(f" {name:<18} ok ({int((time.time() - start) * 1000)}ms)") + print(f" {name:<18} ok ({int((time.monotonic() - start) * 1000)}ms)") finally: with contextlib.suppress(Exception): sb.close() diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 10e279959..870a50941 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "cocoonstack-sandbox" version = "0.1.23" description = "Python SDK for the cocoon sandbox control plane" -requires-python = ">=3.9" +requires-python = ">=3.10" license = { text = "Apache-2.0" } readme = "README.md" @@ -22,10 +22,3 @@ pythonpath = ["."] [tool.mypy] strict = true files = ["cocoonsandbox", "examples"] - -[tool.ruff] -line-length = 120 -target-version = "py39" - -[tool.ruff.lint] -select = ["E", "F", "W", "I", "UP", "B", "SIM"] diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index fbb169ed6..859deb2d9 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -2,17 +2,20 @@ import json import os +import pathlib import resource import socket import threading import time +from collections.abc import Callable from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import BinaryIO, Callable +from typing import BinaryIO import pytest - from cocoonsandbox import Client, Sandbox +FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" + class FakeNode(BaseHTTPRequestHandler): routes = {} @@ -63,12 +66,6 @@ def __init__(self, frames): self._frames = list(frames) self.sent = [] - def __enter__(self): - return self - - def __exit__(self, *exc): - pass - def send(self, op, **fields): self.sent.append((op, fields)) @@ -162,6 +159,21 @@ def answer(): server.close() +@pytest.fixture +def script_server(): + servers = [] + + def start(script, *args) -> str: + server = socket.create_server(("127.0.0.1", 0)) + servers.append(server) + threading.Thread(target=script, args=(server, *args), daemon=True).start() + return f"127.0.0.1:{server.getsockname()[1]}" + + yield start + for server in servers: + server.close() + + @pytest.fixture def black_hole(): sock = socket.socket() diff --git a/sdk/python/tests/test_checkpoint.py b/sdk/python/tests/test_checkpoint.py index 5792db126..1b573ecee 100644 --- a/sdk/python/tests/test_checkpoint.py +++ b/sdk/python/tests/test_checkpoint.py @@ -1,7 +1,6 @@ """Checkpoint claim redirect, mirroring the Client.new redirect in test_fault_matrix.py.""" import pytest - from cocoonsandbox import APIError, Client diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 1572ace17..85c81f5af 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -5,9 +5,8 @@ from http.server import ThreadingHTTPServer import pytest -from conftest import FakeNode, sandbox_at, wait_until - from cocoonsandbox import APIError, Client, Template +from conftest import FakeNode, sandbox_at, wait_until class KeptNode(FakeNode): @@ -344,6 +343,17 @@ def test_a_parked_connection_the_node_closed_is_not_reused(kept_node, fd_floor): assert KeptNode.conns == 2 +def test_close_releases_the_parked_connections(kept_node): + KeptNode.routes[("POST", "/v1/claim")] = lambda body, path: (200, {"id": "sb_1", "token": "tok"}) + with Client(kept_node) as client: + client.new("rt:24.04") + assert KeptNode.closed == 0 + wait_until(lambda: KeptNode.closed == 1, "close left the parked connection open") + client.close() + assert client.new("rt:24.04").id == "sb_1" + assert KeptNode.conns == 2 + + @pytest.fixture def kept_node(): server = ThreadingHTTPServer(("127.0.0.1", 0), KeptNode) diff --git a/sdk/python/tests/test_endpoint.py b/sdk/python/tests/test_endpoint.py index 89869730e..f7a6c5a96 100644 --- a/sdk/python/tests/test_endpoint.py +++ b/sdk/python/tests/test_endpoint.py @@ -1,26 +1,26 @@ +"""Endpoint origin parsing and the TLS handshake timeout.""" + import socket import threading import pytest - from cocoonsandbox import Client, SandboxTimeout from cocoonsandbox.conn import dial_agent from cocoonsandbox.endpoint import _endpoint_url @pytest.mark.parametrize( - ("addr", "scheme", "want"), + ("addr", "want"), [ - ("localhost:7777", "http", "http://localhost:7777"), - ("localhost:443", "https", "https://localhost:443"), - ("https://example.com/", "http", "https://example.com"), - ("http://example.com", "https", "http://example.com"), - ("[::1]:7777", "https", "https://[::1]:7777"), - ("https://[::1]", "http", "https://[::1]"), + ("localhost:7777", "http://localhost:7777"), + ("https://example.com/", "https://example.com"), + ("http://example.com", "http://example.com"), + ("[::1]:7777", "http://[::1]:7777"), + ("https://[::1]", "https://[::1]"), ], ) -def test_endpoint_url(addr: str, scheme: str, want: str) -> None: - assert _endpoint_url(addr, scheme).geturl() == want +def test_endpoint_url(addr: str, want: str) -> None: + assert _endpoint_url(addr).geturl() == want @pytest.mark.parametrize( @@ -38,6 +38,8 @@ def test_endpoint_url(addr: str, scheme: str, want: str) -> None: "https://node:65536", "https://node:bad", "https://node\r\nX: bad", + "n1:7777,n2:7777", + "n1:7777, n2:7777", ], ) def test_endpoint_rejects_invalid_origins(addr: str) -> None: diff --git a/sdk/python/tests/test_fault_matrix.py b/sdk/python/tests/test_fault_matrix.py index 8ae16e486..48b4c72bb 100644 --- a/sdk/python/tests/test_fault_matrix.py +++ b/sdk/python/tests/test_fault_matrix.py @@ -1,24 +1,52 @@ """Fault matrix for the cluster redirect follow and the lookup scatter.""" +import socket +import threading import time import pytest - from cocoonsandbox import APIError, Client -def test_claim_redirect_skips_dead_candidate(spawn_node, dead_addr): +@pytest.mark.parametrize( + ("status", "message"), + [(0, ""), (404, "gone"), (403, "tenant not allowed")], + ids=["dead", "404", "403_definitive"], +) +def test_claim_redirect_skips_a_failed_candidate(spawn_node, dead_addr, status, message): + failed = ( + spawn_node({("POST", "/v1/claim"): lambda body, path: (status, {"error": message})}) if status else dead_addr + ) good = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"id": "sb_ok", "token": "t"})}) - entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [dead_addr, good]})}) + entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [failed, good]})}) sb = Client(entry).new("rt:24.04") assert sb.id == "sb_ok" and sb.owner == good -def test_claim_redirect_skips_404_candidate(spawn_node): - lost = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone"})}) - good = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"id": "sb_2", "token": "t"})}) - entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [lost, good]})}) - assert Client(entry).new("rt:24.04").id == "sb_2" +def test_claim_redirect_stops_when_a_candidate_may_have_claimed(spawn_node): + server = socket.create_server(("127.0.0.1", 0)) + + def drop_reply(): + conn, _ = server.accept() + conn.recv(65536) + conn.close() + + threading.Thread(target=drop_reply, daemon=True).start() + lost = f"127.0.0.1:{server.getsockname()[1]}" + good_calls = [] + good = spawn_node( + {("POST", "/v1/claim"): lambda body, path: good_calls.append(body) or (200, {"id": "x", "token": "t"})} + ) + entry_calls = [] + entry = spawn_node( + {("POST", "/v1/claim"): lambda body, path: entry_calls.append(body) or (200, {"redirect": [lost, good]})} + ) + try: + with pytest.raises(APIError): + Client(entry).new("rt:24.04") + finally: + server.close() + assert not good_calls and len(entry_calls) == 1 def test_claim_redirect_all_candidates_fail_then_origin_fallback_fails(spawn_node): @@ -41,28 +69,29 @@ def entry_claim(body, path): assert len(calls) == 2 and calls[1]["no_redirect"] is True -def test_claim_redirect_all_candidates_fail_then_origin_heals(spawn_node): - full_calls = [] +@pytest.mark.parametrize(("status", "message"), [(429, "full"), (401, "invalid api token")]) +def test_claim_redirect_retryable_candidate_falls_back_to_origin(spawn_node, status, message): + candidate_calls = [] - def full_claim(body, path): - full_calls.append(body) - return 429, {"error": "full"} + def candidate_claim(body, path): + candidate_calls.append(body) + return status, {"error": message} - full = spawn_node({("POST", "/v1/claim"): full_claim}) + candidate = spawn_node({("POST", "/v1/claim"): candidate_claim}) calls = [] def entry_claim(body, path): calls.append(body) if len(calls) == 1: - return 200, {"redirect": [full]} + return 200, {"redirect": [candidate]} return 200, {"id": "sb_healed", "token": "t"} entry = spawn_node({("POST", "/v1/claim"): entry_claim}) sb = Client(entry).new("rt:24.04") assert sb.id == "sb_healed" and sb.owner == entry assert len(calls) == 2 and calls[1]["no_redirect"] is True - assert len(full_calls) == 1 + assert len(candidate_calls) == 1 @pytest.mark.parametrize(("status", "message"), [(409, "no egress"), (403, "tenant not allowed")]) @@ -82,37 +111,6 @@ def entry_claim(body, path): assert len(calls) == 1 -def test_claim_redirect_401_candidate_falls_back_to_origin(spawn_node): - stale_calls = [] - - def stale_claim(body, path): - stale_calls.append(body) - return 401, {"error": "invalid api token"} - - stale = spawn_node({("POST", "/v1/claim"): stale_claim}) - - calls = [] - - def entry_claim(body, path): - calls.append(body) - if len(calls) == 1: - return 200, {"redirect": [stale]} - return 200, {"id": "sb_local", "token": "t"} - - entry = spawn_node({("POST", "/v1/claim"): entry_claim}) - sb = Client(entry).new("rt:24.04") - assert sb.id == "sb_local" and sb.owner == entry - assert len(calls) == 2 and calls[1]["no_redirect"] is True - assert len(stale_calls) == 1 - - -def test_claim_redirect_candidate_definitive_error_still_tries_next_candidate(spawn_node): - forbidden = spawn_node({("POST", "/v1/claim"): lambda body, path: (403, {"error": "tenant not allowed"})}) - good = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"id": "sb_3", "token": "t"})}) - entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [forbidden, good]})}) - assert Client(entry).new("rt:24.04").id == "sb_3" - - def test_delete_template_skips_dead_owner(spawn_node, dead_addr): seen = [] @@ -155,27 +153,14 @@ def ok(body, path): assert touched == ["bad"] -def test_lookup_dead_peer_resolves_fast(spawn_node, dead_addr): - owner = spawn_node({("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {"owner_addr": "10.0.0.9:7777"})}) - entry = spawn_node( - { - ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (404, {"error": "not here"}), - ("GET", "/v1/peers"): lambda body, path: (200, {"peers": [dead_addr, owner]}), - } - ) - start = time.monotonic() - sb = Client(entry, timeout=10.0).lookup("sb_1", "tok") - elapsed = time.monotonic() - start - assert sb.owner == "10.0.0.9:7777" - assert elapsed < 2.0, elapsed - - -def test_lookup_hung_peer_resolves_fast(spawn_node, black_hole): +@pytest.mark.parametrize("peer", ["dead_addr", "black_hole"]) +def test_lookup_unreachable_peer_resolves_fast(spawn_node, request, peer): + unreachable = request.getfixturevalue(peer) owner = spawn_node({("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {"owner_addr": "10.0.0.9:7777"})}) entry = spawn_node( { ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (404, {"error": "not here"}), - ("GET", "/v1/peers"): lambda body, path: (200, {"peers": [black_hole, owner]}), + ("GET", "/v1/peers"): lambda body, path: (200, {"peers": [unreachable, owner]}), } ) start = time.monotonic() @@ -197,7 +182,8 @@ def test_lookup_all_miss(spawn_node, dead_addr): assert exc.value.status == 404 and "no owner found" in exc.value.message -def test_close_releases_a_sandbox_that_moved(spawn_node): +@pytest.mark.parametrize("retired", [False, True], ids=["old_node_404", "old_node_retired"]) +def test_close_releases_a_sandbox_that_moved(spawn_node, dead_addr, retired): released = [] def release(body, path): @@ -208,16 +194,20 @@ def release(body, path): { ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {}), ("POST", "/v1/sandboxes/sb_1/release"): release, + ("GET", "/v1/peers"): lambda body, path: (200, {"peers": []}), } ) - old = spawn_node( - { - ("POST", "/v1/sandboxes/sb_1/release"): lambda body, path: (404, {"error": "unknown sandbox"}), - ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (404, {"error": "not here"}), - ("GET", "/v1/peers"): lambda body, path: (200, {"peers": [adopter]}), - } - ) - Client(old).attach(old, "sb_1", "tok").close() + if retired: + entry, old = adopter, dead_addr + else: + entry = old = spawn_node( + { + ("POST", "/v1/sandboxes/sb_1/release"): lambda body, path: (404, {"error": "unknown sandbox"}), + ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (404, {"error": "not here"}), + ("GET", "/v1/peers"): lambda body, path: (200, {"peers": [adopter]}), + } + ) + Client(entry).attach(old, "sb_1", "tok").close() assert released @@ -242,24 +232,6 @@ def test_a_released_sandbox_still_answers_404(spawn_node): assert exc.value.status == 404 and sb.owner == old -def test_close_follows_a_move_off_a_retired_node(spawn_node, dead_addr): - released = [] - - def release(body, path): - released.append(path) - return (204, {}) - - adopter = spawn_node( - { - ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {}), - ("POST", "/v1/sandboxes/sb_1/release"): release, - ("GET", "/v1/peers"): lambda body, path: (200, {"peers": []}), - } - ) - Client(adopter).attach(dead_addr, "sb_1", "tok").close() - assert released - - def test_close_reports_a_retired_node_when_no_node_holds_the_sandbox(spawn_node, dead_addr): entry = spawn_node( { diff --git a/sdk/python/tests/test_fixtures.py b/sdk/python/tests/test_fixtures.py index f34cb9c69..145f807e9 100644 --- a/sdk/python/tests/test_fixtures.py +++ b/sdk/python/tests/test_fixtures.py @@ -2,13 +2,10 @@ import base64 import json -import pathlib import pytest - from cocoonsandbox import frames - -FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" +from conftest import FIXTURES def fixture_files(prefix): diff --git a/sdk/python/tests/test_frames.py b/sdk/python/tests/test_frames.py index 29a93f75c..a180cd25a 100644 --- a/sdk/python/tests/test_frames.py +++ b/sdk/python/tests/test_frames.py @@ -4,7 +4,6 @@ import json import pytest - from cocoonsandbox import frames diff --git a/sdk/python/tests/test_hardening.py b/sdk/python/tests/test_hardening.py index 286736e03..1f6a3dec0 100644 --- a/sdk/python/tests/test_hardening.py +++ b/sdk/python/tests/test_hardening.py @@ -7,8 +7,6 @@ import time import pytest -from conftest import sandbox_at - from cocoonsandbox import ( APIError, Client, @@ -23,6 +21,7 @@ ) from cocoonsandbox.conn import Conn, dial_agent, remaining_timeout from cocoonsandbox.frames import BULK_CHUNK +from conftest import sandbox_at def test_dial_agent_rejects_control_chars_in_identity(): diff --git a/sdk/python/tests/test_keepalive.py b/sdk/python/tests/test_keepalive.py index 00fb19bc4..536e997bd 100644 --- a/sdk/python/tests/test_keepalive.py +++ b/sdk/python/tests/test_keepalive.py @@ -7,10 +7,9 @@ import threading import pytest -from conftest import accept_upgrade, sandbox_at, wait_until - from cocoonsandbox import APIError, Client, SandboxError, SilkdError from cocoonsandbox.errors import _mark_gone +from conftest import accept_upgrade, sandbox_at, wait_until INPUT_OPS = ("stdin", "stdin_close", "data", "data_end") @@ -184,33 +183,15 @@ def test_an_upload_the_daemon_rejects_at_open_raises_its_error_frame(relay): sb.write_file("/missing/f", bytes(64 << 20)) -def test_a_moved_sandbox_follows_its_new_owner_on_the_next_call(spawn_node): - agent = FakeAgent() - try: - directory = spawn_node( - {("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {"owner_addr": agent.addr})} - ) - old = spawn_node({("GET", "/v1/peers"): lambda body, path: (200, {"peers": [directory]})}) - sb = Client(old).attach(old, "sb_1", "tok") - assert sb.stat("/")["kind"] == "dir" - assert sb.owner == agent.addr - assert sb.exec("echo", "42") == "42\n" - finally: - agent.stop() - - -def test_a_handle_follows_its_sandbox_off_a_retired_node(spawn_node, dead_addr): - agent = FakeAgent() - try: - directory = spawn_node( - {("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {"owner_addr": agent.addr})} - ) - entry = spawn_node({("GET", "/v1/peers"): lambda body, path: (200, {"peers": [directory]})}) - sb = Client(entry).attach(dead_addr, "sb_1", "tok") - assert sb.stat("/")["kind"] == "dir" - assert sb.owner == agent.addr - finally: - agent.stop() +@pytest.mark.parametrize("retired", [False, True], ids=["owner_404", "connection_refused"]) +def test_a_handle_follows_its_moved_sandbox_on_the_next_call(spawn_node, dead_addr, relay, retired): + agent, _ = relay() + directory = spawn_node({("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {"owner_addr": agent.addr})}) + entry = spawn_node({("GET", "/v1/peers"): lambda body, path: (200, {"peers": [directory]})}) + sb = Client(entry).attach(dead_addr if retired else entry, "sb_1", "tok") + assert sb.stat("/")["kind"] == "dir" + assert sb.owner == agent.addr + assert sb.exec("echo", "42") == "42\n" @pytest.mark.parametrize("retired", [False, True], ids=["owner_404", "connection_refused"]) diff --git a/sdk/python/tests/test_proc.py b/sdk/python/tests/test_proc.py index dda18cde2..f6f12e48f 100644 --- a/sdk/python/tests/test_proc.py +++ b/sdk/python/tests/test_proc.py @@ -3,10 +3,9 @@ import threading import pytest -from conftest import FakeConn, sandbox_at - from cocoonsandbox import ProtocolError from cocoonsandbox.frames import BULK_CHUNK +from conftest import FakeConn, sandbox_at class BlockingStdinConn(FakeConn): diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py index a565a72a8..bae509584 100644 --- a/sdk/python/tests/test_stream.py +++ b/sdk/python/tests/test_stream.py @@ -3,63 +3,43 @@ import contextlib import json import socket -import threading import time import pytest -from conftest import accept_upgrade, sandbox_at - from cocoonsandbox import ProtocolError, Sandbox, SandboxTimeout from cocoonsandbox import conn as conn_module from cocoonsandbox import sandbox as sandbox_module +from conftest import accept_upgrade, sandbox_at TIMEOUT = 0.2 -def test_port_stream_outlives_the_client_timeout(): - server = socket.create_server(("127.0.0.1", 0)) - addr = f"127.0.0.1:{server.getsockname()[1]}" +def test_port_stream_outlives_the_client_timeout(script_server): ops: list[str] = [] - threading.Thread(target=serve_port_forward, args=(server, 3 * TIMEOUT, ops), daemon=True).start() - sb = short_timeout_sandbox(addr) - try: - with sb.dial_port(5000) as port: - assert port.recv() == b"late" - assert port.recv() == b"" - finally: - server.close() + sb = short_timeout_sandbox(script_server(serve_port_forward, 3 * TIMEOUT, ops)) + with sb.dial_port(5000) as port: + assert port.recv() == b"late" + assert port.recv() == b"" assert ops == ["port_forward"] -def test_run_timeout_cuts_a_silent_command_and_kills_it(): - server = socket.create_server(("127.0.0.1", 0)) - addr = f"127.0.0.1:{server.getsockname()[1]}" +def test_run_timeout_cuts_a_silent_command_and_kills_it(script_server): kills: list[dict] = [] - threading.Thread(target=serve_started_then_hang, args=(server, 5 * TIMEOUT, kills), daemon=True).start() - sb = short_timeout_sandbox(addr) + sb = short_timeout_sandbox(script_server(serve_started_then_hang, 5 * TIMEOUT, kills)) started = time.monotonic() - try: - with pytest.raises(TimeoutError): - sb.run(["sleep", "9"], timeout=TIMEOUT) - finally: - server.close() + with pytest.raises(TimeoutError): + sb.run(["sleep", "9"], timeout=TIMEOUT) assert time.monotonic() - started < 3 * TIMEOUT assert kills == [{"v": 1, "op": "kill", "pid": 7}], kills -def test_run_timeout_bounds_the_kill_it_sends(monkeypatch): +def test_run_timeout_bounds_the_kill_it_sends(monkeypatch, script_server): monkeypatch.setattr(sandbox_module, "_KILL_WAIT_SECONDS", TIMEOUT) - server = socket.create_server(("127.0.0.1", 0)) - addr = f"127.0.0.1:{server.getsockname()[1]}" kills: list[dict] = [] - threading.Thread(target=serve_started_then_trickle_the_kill_reply, args=(server, kills), daemon=True).start() - sb = short_timeout_sandbox(addr) + sb = short_timeout_sandbox(script_server(serve_started_then_trickle_the_kill_reply, kills)) started = time.monotonic() - try: - with pytest.raises(TimeoutError): - sb.run(["sleep", "9"], timeout=TIMEOUT) - finally: - server.close() + with pytest.raises(TimeoutError): + sb.run(["sleep", "9"], timeout=TIMEOUT) assert time.monotonic() - started < 4 * TIMEOUT assert kills == [{"v": 1, "op": "kill", "pid": 7}], kills @@ -78,21 +58,15 @@ def test_run_rejects_a_non_positive_timeout(): sb.run(["true"], timeout=0) -def test_dial_is_still_bounded_by_the_client_timeout(): - server = socket.create_server(("127.0.0.1", 0)) - addr = f"127.0.0.1:{server.getsockname()[1]}" - threading.Thread(target=serve_silence, args=(server,), daemon=True).start() - sb = short_timeout_sandbox(addr) +def test_dial_is_still_bounded_by_the_client_timeout(script_server): + sb = short_timeout_sandbox(script_server(serve_silence)) started = time.monotonic() - try: - with pytest.raises(SandboxTimeout): - sb.dial_port(5000) - finally: - server.close() + with pytest.raises(SandboxTimeout): + sb.dial_port(5000) assert time.monotonic() - started < 3 * TIMEOUT -@pytest.mark.parametrize(("exc", "want"), [(socket.timeout(), SandboxTimeout), (ConnectionResetError(), ProtocolError)]) +@pytest.mark.parametrize(("exc", "want"), [(TimeoutError(), SandboxTimeout), (ConnectionResetError(), ProtocolError)]) def test_handshake_failures_are_typed(exc, want): assert isinstance(conn_module._handshake_error("agent upgrade", exc), want) diff --git a/sdk/python/tests/test_upgrade.py b/sdk/python/tests/test_upgrade.py index 4ac25de7f..6f0b11920 100644 --- a/sdk/python/tests/test_upgrade.py +++ b/sdk/python/tests/test_upgrade.py @@ -7,9 +7,8 @@ import time import pytest -from conftest import sandbox_at - from cocoonsandbox import APIError, SandboxTimeout +from conftest import sandbox_at REPLIES = { "info": {"type": "info", "version": "fake", "uptime_secs": 0, "procs": 0}, diff --git a/sdk/python/tests/test_volume_controller.py b/sdk/python/tests/test_volume_controller.py index e0215f72d..e41a09663 100644 --- a/sdk/python/tests/test_volume_controller.py +++ b/sdk/python/tests/test_volume_controller.py @@ -1,9 +1,10 @@ +"""The example volume controller's hibernate and resume flows against a fake node.""" + from types import SimpleNamespace import pytest -from conftest import FakeNode - from cocoonsandbox import Client, VolumeError +from conftest import FakeNode from examples.volume_controller import VolumeHooks, hibernate_volume, resume_volume PATH = "/v1/sandboxes/sb_1" diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index e7ddabb5f..e8f233380 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -3,9 +3,8 @@ from collections import UserDict import pytest -from conftest import FakeNode, sandbox_at - from cocoonsandbox import APIError, VolumeError +from conftest import FakeNode, sandbox_at PATH = "/v1/sandboxes/sb_1/volumes" VIEW = { @@ -17,12 +16,7 @@ def test_volume_mapping_is_json_encoded(spawn_node): - seen = [] - - def reply(body, path): - seen.append(body) - return 200, VIEW - + seen, reply = recording() sb = sandbox_at(spawn_node({("PUT", PATH): reply})) sb.set_volumes([UserDict({"name": "data", "mount": "/data"})]) assert seen == [{"volumes": [{"name": "data", "mount": "/data"}]}] @@ -41,12 +35,7 @@ def reply(body, path): ], ) def test_volume_requests_use_owner_and_api_token(spawn_node, method, body, call): - seen = [] - - def reply(body, path): - seen.append(body) - return 200, VIEW - + seen, reply = recording() addr = spawn_node({(method, PATH): reply}) sb = sandbox_at(addr, api_token="operator-token") sb._client.addr = "unused.invalid:7777" @@ -104,12 +93,7 @@ def test_validation_error_preserves_last_view(spawn_node): def test_explicit_empty_set_and_omitted_patch(spawn_node): - seen = [] - - def reply(body, path): - seen.append(body) - return 200, {"volumes_want": [], "volumes": [], "converging": False, "states": []} - + seen, reply = recording({"volumes_want": [], "volumes": [], "converging": False, "states": []}) sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) with pytest.raises(TypeError, match="explicit"): sb.set_volumes(None) @@ -200,13 +184,18 @@ def test_volume_mutations_encode_explicit_device_fields(spawn_node): {"name": "device", "mount": "", "mode": "rw", "attach_only": True}, {"name": "mounted", "attach_only": False}, ] + seen, reply = recording() + sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) + sb.set_volumes(entries) + sb.patch_volumes(add=entries) + assert seen == [{"volumes": entries}, {"add": entries}] + + +def recording(view=VIEW): seen = [] def reply(body, path): seen.append(body) - return 200, VIEW + return 200, view - sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) - sb.set_volumes(entries) - sb.patch_volumes(add=entries) - assert seen == [{"volumes": entries}, {"add": entries}] + return seen, reply diff --git a/sdk/python/tests/test_wire_binding.py b/sdk/python/tests/test_wire_binding.py index a9525541e..7fcf7146d 100644 --- a/sdk/python/tests/test_wire_binding.py +++ b/sdk/python/tests/test_wire_binding.py @@ -2,18 +2,14 @@ import contextlib import json -import pathlib import socket import threading import pytest -from conftest import FakeConn, sandbox_at - from cocoonsandbox import Lsp, Pty, Session from cocoonsandbox.conn import Conn from cocoonsandbox.frames import PROTO_VERSION - -FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" +from conftest import FIXTURES, FakeConn, sandbox_at CASES = [ ( @@ -97,9 +93,6 @@ def recv(self): return {"type": "git_branches", "branches": [], "current": ""} return {"type": "done"} - def recv_until(self, *terminal): - yield self.recv() - @pytest.mark.parametrize("stem,replies,invoke", CASES, ids=lambda c: c if isinstance(c, str) else "") def test_call_site_matches_fixture(monkeypatch, stem, replies, invoke): diff --git a/silkd/src/exec.rs b/silkd/src/exec.rs index 5d7ba6da1..b421fa22b 100644 --- a/silkd/src/exec.rs +++ b/silkd/src/exec.rs @@ -251,7 +251,7 @@ fn effective_path<'a>( }) } -/// Finds a bare program on path like execvp, minus its fall-through past an entry whose exec fails, so std can posix_spawn an absolute path; a relative entry leaves the search to execvp. +/// Resolves a bare program on an all-absolute `path`, so std can posix_spawn it. fn resolve_program(name: &str, path: Option<&str>) -> Option { if name.is_empty() || name.contains('/') { return None; diff --git a/silkd/src/find.rs b/silkd/src/find.rs index 55b4961d0..66e5e0c70 100644 --- a/silkd/src/find.rs +++ b/silkd/src/find.rs @@ -151,7 +151,7 @@ pub async fn find( w: &mut W, path: String, pattern: String, - glob: Option, + glob: Option<&str>, ) -> std::io::Result<()> where R: AsyncBufRead + Unpin, @@ -224,7 +224,7 @@ async fn find_bounded( w: &mut W, path: String, pattern: String, - glob: Option, + glob: Option<&str>, budget_bytes: usize, ) -> std::io::Result<()> where @@ -236,7 +236,7 @@ where Ok(re) => re, Err(e) => return proto::error_frame(w, ErrorKind::BadRequest, e).await, }; - let name_re = match glob.as_deref().filter(|g| !g.is_empty()).map(glob_regex) { + let name_re = match glob.filter(|g| !g.is_empty()).map(glob_regex) { None => None, Some(Ok(re)) => Some(re), Some(Err(e)) => return proto::error_frame(w, ErrorKind::BadRequest, e.to_string()).await, diff --git a/silkd/src/fs.rs b/silkd/src/fs.rs index ae28b38ba..24e0265e2 100644 --- a/silkd/src/fs.rs +++ b/silkd/src/fs.rs @@ -12,22 +12,18 @@ use tokio::sync::mpsc; use crate::proto::{ self, BULK_CHUNK, DirEntry, ErrorKind, FileInfo, FileKind, Response, err_frame, }; +use crate::sysutil; /// Entries per `entries` frame; a worst-case 1.6KiB entry keeps a full batch under MAX_FRAME. pub const LIST_BATCH: usize = 4096; /// Streams client `data` frames into `path` via a temp file renamed only on clean completion. -pub async fn write( - reader: &mut R, - w: &mut W, - path: String, - mode: Option, -) -> io::Result<()> +pub async fn write(reader: &mut R, w: &mut W, path: &str, mode: Option) -> io::Result<()> where R: AsyncBufRead + Unpin, W: AsyncWrite + Unpin, { - let path = Path::new(&path); + let path = Path::new(path); let (tmp, mut file) = match create_tmp(path, mode).await { Ok(created) => created, Err(e) => return err_frame(w, &e, "create").await, @@ -129,8 +125,8 @@ pub async fn write_atomic(path: &Path, bytes: &[u8]) -> io::Result<()> { } /// Reports metadata for `path` (following symlinks). -pub async fn stat(w: &mut W, path: String) -> io::Result<()> { - let meta = match fs::metadata(&path).await { +pub async fn stat(w: &mut W, path: &str) -> io::Result<()> { + let meta = match fs::metadata(path).await { Ok(m) => m, Err(e) => return err_frame(w, &e, "stat").await, }; @@ -150,15 +146,11 @@ pub async fn stat(w: &mut W, path: String) -> io::Result< } /// Creates a directory, with parents when `parents` is set. -pub async fn mkdir( - w: &mut W, - path: String, - parents: bool, -) -> io::Result<()> { +pub async fn mkdir(w: &mut W, path: &str, parents: bool) -> io::Result<()> { let res = if parents { - fs::create_dir_all(&path).await + fs::create_dir_all(path).await } else { - fs::create_dir(&path).await + fs::create_dir(path).await }; match res { Ok(()) => proto::write_frame(w, &Response::Done).await, @@ -167,17 +159,17 @@ pub async fn mkdir( } /// Removes a file, or a directory (recursively when `recursive` is set). -pub async fn rm(w: &mut W, path: String, recursive: bool) -> io::Result<()> { - let meta = match fs::symlink_metadata(&path).await { +pub async fn rm(w: &mut W, path: &str, recursive: bool) -> io::Result<()> { + let meta = match fs::symlink_metadata(path).await { Ok(m) => m, Err(e) => return err_frame(w, &e, "rm").await, }; let res = if meta.is_dir() && recursive { - fs::remove_dir_all(&path).await + fs::remove_dir_all(path).await } else if meta.is_dir() { - fs::remove_dir(&path).await + fs::remove_dir(path).await } else { - fs::remove_file(&path).await + fs::remove_file(path).await }; match res { Ok(()) => proto::write_frame(w, &Response::Done).await, @@ -186,8 +178,8 @@ pub async fn rm(w: &mut W, path: String, recursive: bool) } /// Renames `from` to `to` within the guest filesystem. -pub async fn rename(w: &mut W, from: String, to: String) -> io::Result<()> { - match fs::rename(&from, &to).await { +pub async fn rename(w: &mut W, from: &str, to: &str) -> io::Result<()> { + match fs::rename(from, to).await { Ok(()) => proto::write_frame(w, &Response::Done).await, Err(e) => err_frame(w, &e, "rename").await, } @@ -223,7 +215,7 @@ fn scan_dir(path: &str, tx: &mpsc::Sender>) -> io::Result<()> { fn tmp_name(path: &Path) -> PathBuf { let mut tmp = path.as_os_str().to_os_string(); tmp.push(".silkd-"); - tmp.push(crate::sysutil::tmp_suffix()); + tmp.push(sysutil::tmp_suffix()); tmp.push(".tmp"); PathBuf::from(tmp) } diff --git a/silkd/src/git.rs b/silkd/src/git.rs index 0f6e0e778..1ec4cf828 100644 --- a/silkd/src/git.rs +++ b/silkd/src/git.rs @@ -17,34 +17,34 @@ const NO_EGRESS: &str = "no network on this sandbox; use fs.push to move a repos /// Clones `url` into `path` (optionally a branch, shallow depth), then Done. pub async fn clone( w: &mut W, - url: String, - path: String, - branch: Option, + url: &str, + path: &str, + branch: Option<&str>, depth: Option, - auth: Option, + auth: Option<&str>, ) -> std::io::Result<()> { if !crate::net::has_egress() { return proto::error_frame(w, ErrorKind::Unimplemented, NO_EGRESS).await; } let depth_arg; let mut args: Vec<&str> = vec!["clone"]; - if let Some(b) = &branch { + if let Some(b) = branch { args.extend(["--branch", b]); } if let Some(d) = depth { depth_arg = d.to_string(); args.extend(["--depth", &depth_arg]); } - args.push(&url); - args.push(&path); - // clone has no repo dir yet, so run it from cwd (-C "."). - let out = git(".", auth.as_deref(), &args).await?; + args.push(url); + args.push(path); + // clone has no repo dir yet. + let out = git(".", auth, &args).await?; terminal(w, "clone", &out).await } /// Reports branch, ahead/behind, and per-file status from porcelain v2. -pub async fn status(w: &mut W, path: String) -> std::io::Result<()> { - let out = git(&path, None, &["status", "--porcelain=v2", "--branch"]).await?; +pub async fn status(w: &mut W, path: &str) -> std::io::Result<()> { + let out = git(path, None, &["status", "--porcelain=v2", "--branch"]).await?; if !out.status.success() { return terminal(w, "status", &out).await; } @@ -54,35 +54,35 @@ pub async fn status(w: &mut W, path: String) -> std::io:: /// Stages `files` under `path`. pub async fn add( w: &mut W, - path: String, - files: Vec, + path: &str, + files: &[String], ) -> std::io::Result<()> { let mut args = vec!["add", "--"]; args.extend(files.iter().map(String::as_str)); - let out = git(&path, None, &args).await?; + let out = git(path, None, &args).await?; terminal(w, "add", &out).await } /// Commits staged changes and returns the new commit hash. pub async fn commit( w: &mut W, - path: String, - message: String, - author: String, + path: &str, + message: &str, + author: &str, ) -> std::io::Result<()> { // a fresh guest has no committer identity, so derive one from the author. - let (name, email) = split_author(&author); - let mut cmd = git_cmd(&path, None); + let (name, email) = split_author(author); + let mut cmd = git_cmd(path, None); cmd.env("GIT_COMMITTER_NAME", name) .env("GIT_COMMITTER_EMAIL", email); let out = cmd - .args(["commit", "--author", &author, "-m", &message]) + .args(["commit", "--author", author, "-m", message]) .output() .await?; if !out.status.success() { return terminal(w, "commit", &out).await; } - let rev = git(&path, None, &["rev-parse", "HEAD"]).await?; + let rev = git(path, None, &["rev-parse", "HEAD"]).await?; if !rev.status.success() { return terminal(w, "rev-parse", &rev).await; } @@ -93,26 +93,26 @@ pub async fn commit( /// Runs `push` or `pull` on the current branch behind the egress guard. pub async fn net_verb( w: &mut W, - path: String, - auth: Option, + path: &str, + auth: Option<&str>, verb: &str, ) -> std::io::Result<()> { if !crate::net::has_egress() { return proto::error_frame(w, ErrorKind::Unimplemented, NO_EGRESS).await; } - let out = git(&path, auth.as_deref(), &[verb]).await?; + let out = git(path, auth, &[verb]).await?; terminal(w, verb, &out).await } /// Lists, creates, deletes, or checks out a branch. pub async fn branch( w: &mut W, - path: String, + path: &str, op: GitBranchOp, - name: Option, + name: Option<&str>, ) -> std::io::Result<()> { - let args: Vec<&str> = match (op, &name) { - (GitBranchOp::List, _) => return list_branches(w, &path).await, + let args: Vec<&str> = match (op, name) { + (GitBranchOp::List, _) => return list_branches(w, path).await, (_, None) => { return proto::error_frame(w, ErrorKind::BadRequest, "branch op needs a name").await; } @@ -120,7 +120,7 @@ pub async fn branch( (GitBranchOp::Delete, Some(name)) => vec!["branch", "-D", name], (GitBranchOp::Checkout, Some(name)) => vec!["checkout", name], }; - let out = git(&path, None, &args).await?; + let out = git(path, None, &args).await?; terminal(w, "branch", &out).await } diff --git a/silkd/src/lsp.rs b/silkd/src/lsp.rs index 73579d7d2..6e8f2a5fc 100644 --- a/silkd/src/lsp.rs +++ b/silkd/src/lsp.rs @@ -55,6 +55,7 @@ impl Broker { .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()) + .process_group(0) .kill_on_drop(true); if let Some(dir) = root { cmd.current_dir(dir); @@ -63,10 +64,14 @@ impl Broker { Ok(c) => c, Err(e) => return proto::err_frame(w, &e, "spawn language server").await, }; - let Some(stdio) = child.stdout.take().zip(child.stdin.take()) else { - let _ = child.start_kill(); - return proto::error_frame(w, ErrorKind::Internal, "language server stdio not piped") - .await; + let stdio = sysutil::take_pipe(child.stdout.take(), "stdout") + .and_then(|out| Ok((out, sysutil::take_pipe(child.stdin.take(), "stdin")?))); + let stdio = match stdio { + Ok(stdio) => stdio, + Err(e) => { + let _ = child.start_kill(); + return proto::err_frame(w, &e, "spawn language server").await; + } }; let server = Server { child, @@ -174,6 +179,9 @@ struct Server { } async fn kill(mut server: Server) { + if let Some(pid) = server.child.id() { + sysutil::signal_group(pid, libc::SIGKILL); + } let _ = server.child.start_kill(); let _ = server.child.wait().await; } diff --git a/silkd/src/proto.rs b/silkd/src/proto.rs index 8dde830ac..f9f4da1a1 100644 --- a/silkd/src/proto.rs +++ b/silkd/src/proto.rs @@ -12,7 +12,7 @@ use tokio::io::{ AsyncBufRead, AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, }; -/// Mirrors cocoon-agent's frame cap so a malformed peer can't OOM us. +/// Every client rejects a larger frame, and the cap keeps a malformed peer from OOMing us. pub const MAX_FRAME: usize = 8 * 1024 * 1024; /// Read size for exec, pty and session output. diff --git a/silkd/src/pty.rs b/silkd/src/pty.rs index feca62efc..3c6afd975 100644 --- a/silkd/src/pty.rs +++ b/silkd/src/pty.rs @@ -9,7 +9,7 @@ use tokio::io::AsyncWrite; use tokio::io::unix::AsyncFd; use tokio::process::Command; use tokio::sync::{mpsc, oneshot}; -use tokio::time::timeout; +use tokio::time::{Instant, sleep_until, timeout}; use crate::proc::{Proc, Table}; use crate::proto::{self, ErrorKind, PtyReq, READ_CHUNK, Request, Response}; @@ -17,6 +17,7 @@ use crate::sysutil; /// Bounds the post-exit drain of the master's buffered tail so a stuck fd cannot wedge teardown. const POST_EXIT_DRAIN: Duration = Duration::from_secs(1); +const HUP_GRACE: Duration = Duration::from_secs(2); type Master = Arc>; @@ -129,6 +130,8 @@ async fn pump( let mut frame = Vec::new(); let mut eof = false; let mut disc = false; + let mut gone = false; + let mut kill_at = None; loop { tokio::select! { status = child.wait() => { @@ -137,9 +140,13 @@ async fn pump( stdin.abort(); return code; } - // kill the shell so child.wait publishes its real signalled code. + // hang up like a closed terminal so the shell hangs up its jobs; SIGKILL follows the grace. _ = &mut disc_rx, if !disc => { disc = true; + kill_at = Some(hang_up(child)); + } + _ = sleep_until(kill_at.unwrap_or_else(Instant::now)), if kill_at.is_some() => { + kill_at = None; let _ = child.start_kill(); } readable = master.readable(), if !eof => { @@ -152,10 +159,14 @@ async fn pump( Ok(Ok(0)) | Ok(Err(_)) => eof = true, Ok(Ok(n)) => { proc.emit_bytes(false, &buf[..n]); - if proto::write_chunk_frame(out, &mut frame, "stdout", &buf[..n]).await.is_err() { - let _ = child.start_kill(); + // keep reading after the client is gone: a shell exits only once its tty output drains. + if !gone && proto::write_chunk_frame(out, &mut frame, "stdout", &buf[..n]).await.is_err() { + gone = true; stdin.abort(); - return sysutil::wait_code(child).await; + if !disc { + disc = true; + kill_at = Some(hang_up(child)); + } } } Err(_would_block) => {} @@ -165,6 +176,13 @@ async fn pump( } } +fn hang_up(child: &tokio::process::Child) -> Instant { + if let Some(pid) = child.id() { + sysutil::signal_group(pid, libc::SIGHUP); + } + Instant::now() + HUP_GRACE +} + /// Writes client stdin frames into the master; StdinClose is ignored because a pty outlives input EOF. async fn pump_stdin( master: Master, diff --git a/silkd/src/server.rs b/silkd/src/server.rs index a44297d9b..56771b903 100644 --- a/silkd/src/server.rs +++ b/silkd/src/server.rs @@ -9,7 +9,10 @@ use tokio::task::JoinHandle; use crate::proc::{Chunk, Table, write_chunk}; use crate::proto::{self, ErrorKind, ExecReq, ProcInfo, Request, Response}; -use crate::{exec, find, forward, fs, git, lsp, pty, session, tree, tun, watch}; +use crate::{exec, find, forward, fs, git, lsp, pty, session, sysutil, tree, tun, watch}; + +type Parsed = serde_json::Result; +type Feeder = JoinHandle<(R, Option)>; /// Shared daemon state handed to every connection. pub struct State { @@ -86,10 +89,10 @@ impl State { Request::Attach { pid } => self.attach(&mut reader, writer, pid).await, Request::Exec(e) => return self.exec(e, reader, writer).await, Request::SessionCreate { id, cwd, env } => { - match self.sessions.create(id, cwd, &env).await { + match self.sessions.create(id, cwd.as_deref(), &env).await { Ok(id) => proto::write_frame(writer, &Response::SessionCreated { id }).await, Err(e) => { - let kind = crate::exec::spawn_kind(&e); + let kind = exec::spawn_kind(&e); proto::error_frame(writer, kind, format!("session create: {e}")).await } } @@ -110,20 +113,20 @@ impl State { proto::error_frame(writer, ErrorKind::NotFound, "no such session").await } } - Request::FsWrite { path, mode } => fs::write(&mut reader, writer, path, mode).await, + Request::FsWrite { path, mode } => fs::write(&mut reader, writer, &path, mode).await, Request::FsRead { path } => fs::read(writer, path).await, Request::FsList { path } => fs::list(writer, path).await, - Request::FsStat { path } => fs::stat(writer, path).await, - Request::FsMkdir { path, parents } => fs::mkdir(writer, path, parents).await, - Request::FsRm { path, recursive } => fs::rm(writer, path, recursive).await, - Request::FsRename { from, to } => fs::rename(writer, from, to).await, - Request::FsPush { dest } => tree::push(&mut reader, writer, dest).await, - Request::FsPull { path } => tree::pull(writer, path).await, + Request::FsStat { path } => fs::stat(writer, &path).await, + Request::FsMkdir { path, parents } => fs::mkdir(writer, &path, parents).await, + Request::FsRm { path, recursive } => fs::rm(writer, &path, recursive).await, + Request::FsRename { from, to } => fs::rename(writer, &from, &to).await, + Request::FsPush { dest } => tree::push(&mut reader, writer, &dest).await, + Request::FsPull { path } => tree::pull(writer, &path).await, Request::FsFind { path, pattern, glob, - } => find::find(&mut reader, writer, path, pattern, glob).await, + } => find::find(&mut reader, writer, path, pattern, glob.as_deref()).await, Request::FsReplace { files, pattern, @@ -168,18 +171,32 @@ impl State { branch, depth, auth, - } => git::clone(writer, url, path, branch, depth, auth).await, - Request::GitStatus { path } => git::status(writer, path).await, - Request::GitAdd { path, files } => git::add(writer, path, files).await, + } => { + git::clone( + writer, + &url, + &path, + branch.as_deref(), + depth, + auth.as_deref(), + ) + .await + } + Request::GitStatus { path } => git::status(writer, &path).await, + Request::GitAdd { path, files } => git::add(writer, &path, &files).await, Request::GitCommit { path, message, author, - } => git::commit(writer, path, message, author).await, - Request::GitPush { path, auth } => git::net_verb(writer, path, auth, "push").await, - Request::GitPull { path, auth } => git::net_verb(writer, path, auth, "pull").await, + } => git::commit(writer, &path, &message, &author).await, + Request::GitPush { path, auth } => { + git::net_verb(writer, &path, auth.as_deref(), "push").await + } + Request::GitPull { path, auth } => { + git::net_verb(writer, &path, auth.as_deref(), "pull").await + } Request::GitBranch { path, action, name } => { - git::branch(writer, path, action, name).await + git::branch(writer, &path, action, name.as_deref()).await } Request::Stdin { .. } | Request::StdinClose @@ -201,7 +218,6 @@ impl State { R: AsyncBufRead + Unpin + Send + 'static, W: AsyncWrite + Unpin, { - // validated ahead of the session/process split so an empty argv errors on both paths. if e.argv.is_empty() { proto::error_frame(writer, ErrorKind::BadRequest, "argv must not be empty").await?; return Ok((reader, None)); @@ -261,7 +277,7 @@ impl State { }; // an exited pid may already be recycled, so signalling it would hit an unrelated process. if proc.exit_code().is_none() { - crate::sysutil::signal_group(proc.pid, signal.unwrap_or(libc::SIGKILL)); + sysutil::signal_group(proc.pid, signal.unwrap_or(libc::SIGKILL)); } proto::write_frame(w, &Response::Done).await } @@ -329,9 +345,6 @@ impl Default for State { } } -type Parsed = serde_json::Result; -type Feeder = JoinHandle<(R, Option)>; - fn spawn_feeder(reader: R) -> (mpsc::Receiver, Feeder) where R: AsyncBufRead + Unpin + Send + 'static, diff --git a/silkd/src/session.rs b/silkd/src/session.rs index 7d12b1159..957f985ba 100644 --- a/silkd/src/session.rs +++ b/silkd/src/session.rs @@ -31,7 +31,7 @@ impl Table { pub async fn create( &self, id: Option, - cwd: Option, + cwd: Option<&str>, env: &HashMap, ) -> io::Result { let id = match id { @@ -47,7 +47,7 @@ impl Table { let mut cmd = Command::new("bash"); cmd.env_clear().envs(sysutil::base_env().iter().copied()); sysutil::apply_claim_env(&mut cmd); - if let Some(dir) = &cwd { + if let Some(dir) = cwd { cmd.current_dir(dir); } let mut child = cmd @@ -60,8 +60,8 @@ impl Table { .kill_on_drop(true) .spawn()?; let pid = child.id().unwrap_or(0); - let stdin = take_pipe(child.stdin.take(), "stdin")?; - let stdout = take_pipe(child.stdout.take(), "stdout")?; + let stdin = sysutil::take_pipe(child.stdin.take(), "stdin")?; + let stdout = sysutil::take_pipe(child.stdout.take(), "stdout")?; let session = Arc::new(Session { pid, io: tokio::sync::Mutex::new(Io { @@ -89,7 +89,6 @@ impl Table { io.converse::(&init, None).await?; } - // reserve the id under the table lock so a concurrent create with the same id sees AlreadyExists. let mut map = sysutil::lock(&self.inner); if map.contains_key(&id) { return Err(io::Error::new( @@ -278,11 +277,6 @@ async fn emit(out: &mut Option<&mut W>, frame: &mut Vec(pipe: Option, name: &str) -> io::Result { - pipe.ok_or_else(|| io::Error::other(format!("child {name} not piped"))) -} - /// POSIX single-quote quoting: wraps `s` in '…' onto `out`, reopening around embedded quotes. fn shell_quote_into(out: &mut String, s: &str) { out.push('\''); diff --git a/silkd/src/sysutil.rs b/silkd/src/sysutil.rs index 01927695c..9174ce853 100644 --- a/silkd/src/sysutil.rs +++ b/silkd/src/sysutil.rs @@ -12,7 +12,7 @@ use std::process::ExitStatus; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{LazyLock, Mutex}; -use nix::fcntl::{FcntlArg, FdFlag, OFlag, fcntl}; +use nix::fcntl::{FcntlArg, OFlag, fcntl}; use tokio::io::unix::AsyncFd; use tokio::process::Command; @@ -130,26 +130,38 @@ pub fn start_in(cmd: &mut Command, dir: &OwnedFd) { // SAFETY: fchdir is async-signal-safe and `dir` outlives the spawn that runs this hook. unsafe { cmd.pre_exec(move || { - nix::unistd::fchdir(std::os::fd::BorrowedFd::borrow_raw(fd)) - .map_err(std::io::Error::from) + nix::unistd::fchdir(BorrowedFd::borrow_raw(fd)).map_err(std::io::Error::from) }) }; } -/// Opens a pty, returning the (master, slave) fds; the master is non-blocking for async I/O. +/// Opens a pty, returning the (master, slave) fds; both are close-on-exec from birth and the master is non-blocking. pub fn openpty(cols: u16, rows: u16) -> std::io::Result<(OwnedFd, OwnedFd)> { - let ws = nix::pty::Winsize { - ws_row: rows, - ws_col: cols, - ws_xpixel: 0, - ws_ypixel: 0, - }; - let pty = nix::pty::openpty(&ws, None)?; - fcntl(&pty.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC))?; - fcntl(&pty.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC))?; - let flags = OFlag::from_bits_retain(fcntl(&pty.master, FcntlArg::F_GETFL)?); - fcntl(&pty.master, FcntlArg::F_SETFL(flags | OFlag::O_NONBLOCK))?; - Ok((pty.master, pty.slave)) + let master = nix::pty::posix_openpt(OFlag::O_RDWR | OFlag::O_NOCTTY | OFlag::O_CLOEXEC)?; + nix::pty::grantpt(&master)?; + nix::pty::unlockpt(&master)?; + let slave = nix::fcntl::open( + pty_name(&master)?.as_str(), + OFlag::O_RDWR | OFlag::O_NOCTTY | OFlag::O_CLOEXEC, + nix::sys::stat::Mode::empty(), + )?; + let master = OwnedFd::from(master); + set_winsize(master.as_raw_fd(), cols, rows)?; + set_nonblocking(master.as_fd(), true)?; + Ok((master, slave)) +} + +#[cfg(target_os = "linux")] +fn pty_name(master: &nix::pty::PtyMaster) -> nix::Result { + nix::pty::ptsname_r(master) +} + +#[cfg(not(target_os = "linux"))] +fn pty_name(master: &nix::pty::PtyMaster) -> nix::Result { + static PTSNAME: Mutex<()> = Mutex::new(()); + let _serial = lock(&PTSNAME); + // SAFETY: PTSNAME serializes every ptsname call in silkd, so its static buffer is copied before another call reuses it. + unsafe { nix::pty::ptsname(master) } } pub fn watch_fd(fd: T) -> std::io::Result> { @@ -200,6 +212,11 @@ pub async fn wait_code(child: &mut tokio::process::Child) -> i32 { child.wait().await.map_or(-1, exit_code) } +/// Unwraps a piped child stdio handle; absent means the spawn contract broke. +pub fn take_pipe(pipe: Option, name: &str) -> std::io::Result { + pipe.ok_or_else(|| std::io::Error::other(format!("child {name} not piped"))) +} + /// Opens tun device `name` with a virtio-net header leading every packet. #[cfg(target_os = "linux")] pub fn tun_open(name: &str) -> std::io::Result { @@ -496,6 +513,18 @@ fn check(rc: libc::c_int) -> std::io::Result<()> { mod tests { use super::*; + #[test] + fn openpty_fds_are_close_on_exec_and_the_master_non_blocking() { + let (master, slave) = openpty(80, 24).unwrap(); + for fd in [&master, &slave] { + let fd_flags = + nix::fcntl::FdFlag::from_bits_retain(fcntl(fd, FcntlArg::F_GETFD).unwrap()); + assert!(fd_flags.contains(nix::fcntl::FdFlag::FD_CLOEXEC)); + } + let fl = OFlag::from_bits_retain(fcntl(&master, FcntlArg::F_GETFL).unwrap()); + assert!(fl.contains(OFlag::O_NONBLOCK)); + } + #[test] fn base_env_has_path_and_term() { let env = base_env(); diff --git a/silkd/src/tree.rs b/silkd/src/tree.rs index 497f1dba8..7fd1b8ded 100644 --- a/silkd/src/tree.rs +++ b/silkd/src/tree.rs @@ -21,28 +21,28 @@ const DIR_FLAGS: OFlag = OFlag::O_DIRECTORY .union(OFlag::O_CLOEXEC); /// Extracts a client tar stream into `dest`, creating it; a stream failure leaves `dest` unchanged. -pub async fn push(reader: &mut R, w: &mut W, dest: String) -> io::Result<()> +pub async fn push(reader: &mut R, w: &mut W, dest: &str) -> io::Result<()> where R: AsyncBufRead + Unpin, W: AsyncWrite + Unpin, { - if let Err(e) = tokio::fs::create_dir_all(&dest).await { + if let Err(e) = tokio::fs::create_dir_all(dest).await { return err_frame(w, &e, "mkdir dest").await; } let name = format!(".silkd-push-{}", sysutil::tmp_suffix()); - let (dest_fd, staging) = match stage_dir(&dest, &name) { + let (dest_fd, staging) = match stage_dir(dest, &name) { Ok(fds) => fds, Err(e) => return err_frame(w, &e, "stage push").await, }; let res = push_staged(reader, w, staging, dest_fd).await; // a failed push can leave a large partial tree, so cleanup runs on every exit path. - let _ = tokio::fs::remove_dir_all(Path::new(&dest).join(&name)).await; + let _ = tokio::fs::remove_dir_all(Path::new(dest).join(&name)).await; res } /// Streams `path` back as a tar carrying the entry under its own basename. -pub async fn pull(w: &mut W, path: String) -> io::Result<()> { - let p = Path::new(&path); +pub async fn pull(w: &mut W, path: &str) -> io::Result<()> { + let p = Path::new(path); let (parent, name) = match (p.parent(), p.file_name()) { (Some(par), Some(n)) => (par, n), _ => return proto::error_frame(w, ErrorKind::BadRequest, "invalid path").await, @@ -68,8 +68,9 @@ pub async fn pull(w: &mut W, path: String) -> io::Result< Ok(pair) => pair, Err(e) => return err_frame(w, &e, "spawn tar").await, }; - let Some(mut out) = child.stdout.take() else { - return err_frame(w, &io::Error::other("tar stdout not piped"), "spawn tar").await; + let mut out = match sysutil::take_pipe(child.stdout.take(), "stdout") { + Ok(out) => out, + Err(e) => return err_frame(w, &e, "spawn tar").await, }; if let Err(e) = proto::stream_data_frames(&mut out, w).await? { @@ -99,8 +100,9 @@ where Ok(pair) => pair, Err(e) => return err_frame(w, &e, "spawn tar").await, }; - let Some(mut sink) = child.stdin.take() else { - return err_frame(w, &io::Error::other("tar stdin not piped"), "spawn tar").await; + let mut sink = match sysutil::take_pipe(child.stdin.take(), "stdin") { + Ok(sink) => sink, + Err(e) => return err_frame(w, &e, "spawn tar").await, }; let feed = proto::feed_data_frames(reader, &mut sink).await; @@ -132,10 +134,7 @@ fn spawn_tar( mut cmd: Command, ) -> io::Result<(tokio::process::Child, tokio::task::JoinHandle>)> { let mut child = cmd.stderr(Stdio::piped()).kill_on_drop(true).spawn()?; - let stderr = child - .stderr - .take() - .ok_or_else(|| io::Error::other("tar stderr not piped"))?; + let stderr = sysutil::take_pipe(child.stderr.take(), "stderr")?; Ok((child, tokio::spawn(drain(stderr)))) } diff --git a/silkd/src/watch.rs b/silkd/src/watch.rs index c68594984..ae4decec8 100644 --- a/silkd/src/watch.rs +++ b/silkd/src/watch.rs @@ -7,8 +7,9 @@ use tokio::sync::mpsc; use crate::proto::{self, ErrorKind, EventKind, Response}; const OVERFLOW_MESSAGE: &str = "watch event queue overflow"; +const EVENT_QUEUE: usize = 256; -/// Events written per syscall; batching keeps the queue far from its 256-slot cap. +/// Events written per syscall; batching keeps the queue far from its cap. const EVENT_BATCH: usize = 64; /// Watches `path`, writing `ready`, ordered events, or a terminal error until disconnect. @@ -22,7 +23,7 @@ where R: AsyncBufRead + Unpin, W: AsyncWrite + Unpin, { - let (tx, mut rx) = mpsc::channel::(256); + let (tx, mut rx) = mpsc::channel::(EVENT_QUEUE); // arming walks the tree and adds a watch per directory, so it must not hold a runtime worker let armed = tokio::task::spawn_blocking(move || arm(tx, &path, recursive)).await; let _watcher = match armed { diff --git a/silkd/tests/common/mod.rs b/silkd/tests/common/mod.rs index 3d5b2f892..f566be0f7 100644 --- a/silkd/tests/common/mod.rs +++ b/silkd/tests/common/mod.rs @@ -2,6 +2,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used)] #![allow(dead_code)] +use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -11,6 +12,7 @@ use silkd::server::State; use tokio::io::{ AsyncBufReadExt, AsyncWriteExt, BufReader, DuplexStream, Lines, ReadHalf, WriteHalf, }; +use tokio::net::TcpListener; use tokio::task::JoinHandle; use tokio::time::timeout; @@ -28,6 +30,19 @@ pub fn connect(state: &Arc) -> (FrameWriter, FrameLines, JoinHandle, + first: Value, +) -> (FrameWriter, FrameLines, JoinHandle>) { + let (mut cw, lines, handle) = connect(state); + send(&mut cw, first).await; + (cw, lines, handle) +} + +pub async fn under_deadline(fut: impl Future) -> T { + timeout(DEADLINE, fut).await.expect("deadline") +} + pub async fn send(cw: &mut FrameWriter, frame: Value) { let mut line = serde_json::to_vec(&frame).expect("encode frame"); line.push(b'\n'); @@ -43,6 +58,12 @@ pub async fn next_frame(lines: &mut FrameLines) -> Value { serde_json::from_str(&line).unwrap_or_else(|e| panic!("bad frame {line:?}: {e}")) } +pub async fn expect_frame(lines: &mut FrameLines, kind: &str) -> Value { + let frame = next_frame(lines).await; + assert_type(&frame, kind); + frame +} + pub async fn frames_until(lines: &mut FrameLines, pred: impl Fn(&Value) -> bool) -> Vec { let mut frames = Vec::new(); loop { @@ -69,6 +90,11 @@ pub fn assert_error(frame: &Value, kind: &str) { assert_eq!(frame["kind"], kind, "{frame}"); } +#[track_caller] +pub fn assert_type(frame: &Value, kind: &str) { + assert_eq!(type_of(frame), kind, "{frame}"); +} + pub async fn one(state: &Arc, request_line: &str) -> Vec { request_on(state, &[request_line.to_string()]).await } @@ -77,6 +103,47 @@ pub async fn exchange(lines: &[String]) -> Vec { request_on(&Arc::new(State::new()), lines).await } +pub fn proc_entry(ps: &[Value], pid: u64) -> Option<&Value> { + ps[0]["procs"] + .as_array() + .unwrap() + .iter() + .find(|p| p["pid"].as_u64() == Some(pid)) +} + +pub async fn bind_local() -> (TcpListener, u16) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let port = listener.local_addr().expect("addr").port(); + (listener, port) +} + +pub async fn fs_push(dest: &Path, archive: &[u8]) -> Vec { + let mut lines = vec![json!({"op":"fs_push","dest":dest.to_str().unwrap()}).to_string()]; + lines.extend(data_frames(archive)); + exchange(&lines).await +} + +pub async fn wait_for_path(path: &Path) { + under_deadline(async { + while !path.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; +} + +pub fn alive(pid: u32) -> bool { + // SAFETY: kill(2) with signal 0 only probes whether the pid exists; it takes no pointers. + if unsafe { libc::kill(pid as libc::pid_t, 0) } != 0 { + return false; + } + !std::fs::read_to_string(format!("/proc/{pid}/stat")).is_ok_and(|stat| { + stat.rsplit(')') + .next() + .is_some_and(|rest| rest.trim_start().starts_with('Z')) + }) +} + pub fn type_of(frame: &Value) -> &str { frame["type"].as_str().unwrap_or("") } diff --git a/silkd/tests/exec_e2e.rs b/silkd/tests/exec_e2e.rs index 5a0f81875..e7027e9aa 100644 --- a/silkd/tests/exec_e2e.rs +++ b/silkd/tests/exec_e2e.rs @@ -11,8 +11,8 @@ use silkd::server::State; use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; use common::{ - assert_error, b64, connect, exchange, frames_until, next_frame, one, roundtrip, send, - stdout_body, type_of, + alive, assert_error, b64, connect_with, exchange, frames_until, next_frame, one, proc_entry, + roundtrip, send, stdout_body, type_of, wait_for_path, }; async fn detached_pid(state: &Arc, script: &str) -> u64 { @@ -27,8 +27,7 @@ async fn detached_pid(state: &Arc, script: &str) -> u64 { } async fn attach_frames(state: &Arc, pid: u64) -> Vec { - let (mut cw, mut lines, _) = connect(state); - send(&mut cw, json!({"op":"attach","pid":pid})).await; + let (_cw, mut lines, _) = connect_with(state, json!({"op":"attach","pid":pid})).await; frames_until(&mut lines, |f| { matches!(type_of(f), "exit" | "done" | "error") }) @@ -49,12 +48,7 @@ async fn wait_for_exit(state: &Arc, pid: u64) -> Vec { async fn wait_listed(state: &Arc, pid: u64) -> Value { for _ in 0..250 { let ps = one(state, r#"{"op":"ps"}"#).await; - if let Some(p) = ps[0]["procs"] - .as_array() - .unwrap() - .iter() - .find(|p| p["pid"].as_u64() == Some(pid)) - { + if let Some(p) = proc_entry(&ps, pid) { return p.clone(); } tokio::time::sleep(Duration::from_millis(20)).await; @@ -217,8 +211,7 @@ async fn attach_keeps_a_tail_emitted_during_the_replay() { async fn attach_ends_when_its_client_hangs_up() { let state = Arc::new(State::new()); let pid = detached_pid(&state, "sleep 30").await; - let (mut cw, lines, handle) = connect(&state); - send(&mut cw, json!({"op":"attach","pid":pid})).await; + let (cw, lines, handle) = connect_with(&state, json!({"op":"attach","pid":pid})).await; drop(lines); drop(cw); @@ -349,9 +342,8 @@ async fn detached_exec_is_listed_then_logs_replay_output_and_exit() { #[tokio::test] async fn disconnect_during_drain_publishes_real_exit_code() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - send( - &mut cw, + let (cw, mut lines, _) = connect_with( + &state, json!({ "op": "exec", "argv": ["/bin/sh", "-c", "{ sleep 0.5; echo late; sleep 30; } & exit 7"] @@ -385,9 +377,8 @@ async fn disconnect_during_drain_publishes_real_exit_code() { #[tokio::test] async fn stdin_held_by_a_forked_child_does_not_wedge_the_next_rpc() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send( - &mut cw, + let (mut cw, mut lines, handle) = connect_with( + &state, json!({ "op": "exec", "argv": ["/bin/sh", "-c", "exec 3<&0; sleep 8 <&3 3<&- & exit 0"] @@ -415,9 +406,8 @@ async fn a_client_gone_mid_output_releases_the_stdin_a_forked_child_holds() { let dir = tempfile::tempdir().unwrap(); let out = dir.path().join("count"); let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - send( - &mut cw, + let (mut cw, mut lines, _) = connect_with( + &state, json!({ "op": "exec", "argv": ["/bin/sh", "-c", "exec 3<&0; perl -e 'setpgrp; exec @ARGV' /bin/sh -c 'sleep 1; wc -c <&3 >\"$OUT.tmp\"; mv \"$OUT.tmp\" \"$OUT\"' >/dev/null 2>&1 & while :; do echo x; sleep 0.05; done"], @@ -433,14 +423,7 @@ async fn a_client_gone_mid_output_releases_the_stdin_a_forked_child_holds() { drop(lines); drop(cw); - let deadline = tokio::time::Instant::now() + Duration::from_secs(8); - while !out.exists() { - assert!( - tokio::time::Instant::now() < deadline, - "the forked child never saw stdin end" - ); - tokio::time::sleep(Duration::from_millis(50)).await; - } + wait_for_path(&out).await; let read: usize = std::fs::read_to_string(&out) .unwrap() .trim() @@ -451,10 +434,3 @@ async fn a_client_gone_mid_output_releases_the_stdin_a_forked_child_holds() { "the forked child read {read} bytes: stdin kept flowing after its client left" ); } - -fn alive(pid: u32) -> bool { - std::process::Command::new("/bin/sh") - .args(["-c", &format!("kill -0 {pid} 2>/dev/null")]) - .status() - .is_ok_and(|s| s.success()) -} diff --git a/silkd/tests/find_e2e.rs b/silkd/tests/find_e2e.rs index 66edc3bb3..1679322a3 100644 --- a/silkd/tests/find_e2e.rs +++ b/silkd/tests/find_e2e.rs @@ -10,11 +10,10 @@ use serde_json::{Value, json}; use silkd::server::State; use tokio::io::AsyncWriteExt; -use common::{assert_error, connect, frames_until, request, send, type_of}; +use common::{assert_error, connect_with, expect_frame, frames_until, request, send, type_of}; async fn find_frames(req: Value) -> Vec { - let (mut cw, mut out, handle) = connect(&Arc::new(State::new())); - send(&mut cw, req).await; + let (mut cw, mut out, handle) = connect_with(&Arc::new(State::new()), req).await; let frames = frames_until(&mut out, |f| matches!(type_of(f), "done" | "error")).await; cw.shutdown().await.unwrap(); handle.await.unwrap().unwrap(); @@ -27,9 +26,8 @@ async fn find_serves_a_request_pipelined_behind_it_after_done() { tokio::fs::write(dir.path().join("a.txt"), "needle\n") .await .unwrap(); - let (mut cw, mut out, handle) = connect(&Arc::new(State::new())); - send( - &mut cw, + let (mut cw, mut out, handle) = connect_with( + &Arc::new(State::new()), json!({"op":"fs_find","path":dir.path().to_str().unwrap(),"pattern":"needle"}), ) .await; @@ -37,7 +35,7 @@ async fn find_serves_a_request_pipelined_behind_it_after_done() { let frames = frames_until(&mut out, |f| matches!(type_of(f), "done" | "error")).await; assert_eq!(type_of(frames.last().unwrap()), "done", "{frames:?}"); assert!(frames.iter().any(|f| type_of(f) == "match"), "{frames:?}"); - assert_eq!(type_of(&common::next_frame(&mut out).await), "info"); + expect_frame(&mut out, "info").await; cw.shutdown().await.unwrap(); handle.await.unwrap().unwrap(); } diff --git a/silkd/tests/forward_e2e.rs b/silkd/tests/forward_e2e.rs index b6b3fd716..c8a0cd51a 100644 --- a/silkd/tests/forward_e2e.rs +++ b/silkd/tests/forward_e2e.rs @@ -9,18 +9,15 @@ use std::time::{Duration, Instant}; use serde_json::json; use silkd::server::State; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpListener; use tokio::sync::oneshot; -use tokio::time::timeout; use common::{ - DEADLINE, FrameLines, FrameWriter, assert_error, b64, connect, decode, next_frame, send, - type_of, + FrameLines, FrameWriter, assert_error, b64, bind_local, connect, connect_with, decode, + expect_frame, next_frame, send, type_of, under_deadline, }; async fn echo_listener() -> u16 { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let port = listener.local_addr().expect("addr").port(); + let (listener, port) = bind_local().await; tokio::spawn(async move { while let Ok((mut conn, _)) = listener.accept().await { tokio::spawn(async move { @@ -44,14 +41,12 @@ async fn echo_listener() -> u16 { async fn forwarded(state: &Arc, port: u16) -> (FrameWriter, FrameLines) { let (mut cw, mut lines, _) = connect(state); send(&mut cw, json!({"v":1,"op":"port_forward","port":port})).await; - let ready = next_frame(&mut lines).await; - assert_eq!(type_of(&ready), "ready", "got {ready}"); + expect_frame(&mut lines, "ready").await; (cw, lines) } async fn request_listener(request: usize) -> u16 { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let port = listener.local_addr().expect("addr").port(); + let (listener, port) = bind_local().await; tokio::spawn(async move { let (mut conn, _) = listener.accept().await.expect("accept"); let mut buf = vec![0u8; request]; @@ -65,8 +60,7 @@ async fn request_listener(request: usize) -> u16 { } async fn half_close_listener() -> (u16, oneshot::Receiver>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let port = listener.local_addr().expect("addr").port(); + let (listener, port) = bind_local().await; let (tx, rx) = oneshot::channel(); tokio::spawn(async move { let (conn, _) = listener.accept().await.expect("accept"); @@ -82,37 +76,32 @@ async fn half_close_listener() -> (u16, oneshot::Receiver>) { #[tokio::test] async fn forward_round_trips_and_done_on_server_close() { - timeout(DEADLINE, async { + under_deadline(async { let port = echo_listener().await; let state = Arc::new(State::new()); let (mut cw, mut lines) = forwarded(&state, port).await; send(&mut cw, json!({"v":1,"op":"data","data":b64(b"hi")})).await; - let echoed = next_frame(&mut lines).await; - assert_eq!(type_of(&echoed), "data", "got {echoed}"); + let echoed = expect_frame(&mut lines, "data").await; assert_eq!(decode(&echoed), b"hi"); send(&mut cw, json!({"v":1,"op":"data_end"})).await; - let done = next_frame(&mut lines).await; - assert_eq!(type_of(&done), "done", "got {done}"); + expect_frame(&mut lines, "done").await; }) - .await - .expect("test deadline"); + .await; } #[tokio::test] async fn forward_accepts_client_writes_after_the_guest_half_closes() { - timeout(DEADLINE, async { + under_deadline(async { let (port, got) = half_close_listener().await; let state = Arc::new(State::new()); let (mut cw, mut lines) = forwarded(&state, port).await; - let greeting = next_frame(&mut lines).await; - assert_eq!(type_of(&greeting), "data", "got {greeting}"); + let greeting = expect_frame(&mut lines, "data").await; assert_eq!(decode(&greeting), b"greeting"); - let done = next_frame(&mut lines).await; - assert_eq!(type_of(&done), "done", "got {done}"); + expect_frame(&mut lines, "done").await; send( &mut cw, @@ -127,8 +116,7 @@ async fn forward_accepts_client_writes_after_the_guest_half_closes() { "guest lost the client's post-done bytes" ); }) - .await - .expect("test deadline"); + .await; } #[tokio::test] @@ -136,7 +124,7 @@ async fn forward_bidirectional_bulk_no_deadlock() { const CHUNK: usize = 3072; const FRAMES: usize = 400; - timeout(DEADLINE, async { + under_deadline(async { let port = echo_listener().await; let state = Arc::new(State::new()); let (mut cw, mut lines) = forwarded(&state, port).await; @@ -159,44 +147,38 @@ async fn forward_bidirectional_bulk_no_deadlock() { writer.await.expect("writer"); assert_eq!(got, CHUNK * FRAMES); }) - .await - .expect("test deadline"); + .await; } #[tokio::test] async fn forward_refused_port_is_not_found() { - timeout(DEADLINE, async { + under_deadline(async { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send(&mut cw, json!({"v":1,"op":"port_forward","port":1})).await; + let (_cw, mut lines, _) = + connect_with(&state, json!({"v":1,"op":"port_forward","port":1})).await; let err = next_frame(&mut lines).await; assert_error(&err, "not_found"); }) - .await - .expect("test deadline"); + .await; } #[tokio::test] async fn a_request_during_forward_ends_its_input_and_runs_after_it() { - timeout(DEADLINE, async { + under_deadline(async { let port = echo_listener().await; let state = Arc::new(State::new()); let (mut cw, mut lines) = forwarded(&state, port).await; send(&mut cw, json!({"v":1,"op":"ps"})).await; - let done = next_frame(&mut lines).await; - assert_eq!(type_of(&done), "done", "got {done}"); - let procs = next_frame(&mut lines).await; - assert_eq!(type_of(&procs), "procs", "got {procs}"); + expect_frame(&mut lines, "done").await; + expect_frame(&mut lines, "procs").await; }) - .await - .expect("test deadline"); + .await; } #[tokio::test] async fn forward_split_requests_do_not_wait_on_delayed_acks() { - timeout(DEADLINE, async { + under_deadline(async { let port = request_listener(8192 + 1024).await; let state = Arc::new(State::new()); let (mut cw, mut lines) = forwarded(&state, port).await; @@ -221,6 +203,5 @@ async fn forward_split_requests_do_not_wait_on_delayed_acks() { "20 split requests took {elapsed:?}: the guest-side write waits on delayed ACKs" ); }) - .await - .expect("test deadline"); + .await; } diff --git a/silkd/tests/git_e2e.rs b/silkd/tests/git_e2e.rs index 7fc03ee13..de844ed38 100644 --- a/silkd/tests/git_e2e.rs +++ b/silkd/tests/git_e2e.rs @@ -136,8 +136,9 @@ async fn commit_failure_surfaces_git_error() { #[tokio::test] async fn clone_lane_behavior() { silkd::net::override_egress_for_tests(Some(false)); + let refused = tempfile::tempdir().unwrap(); let none = request(json!({ - "op":"git_clone","url":"https://example.com/x.git","path":"/tmp/silkd-x" + "op":"git_clone","url":"https://example.com/x.git","path":refused.path().join("x").to_str().unwrap() })) .await; assert_error(&none[0], "unimplemented"); diff --git a/silkd/tests/keepalive_e2e.rs b/silkd/tests/keepalive_e2e.rs index 345548a33..2f928ddff 100644 --- a/silkd/tests/keepalive_e2e.rs +++ b/silkd/tests/keepalive_e2e.rs @@ -11,7 +11,8 @@ use tokio::io::AsyncWriteExt; use tokio::time::timeout; use common::{ - DEADLINE, FrameLines, assert_error, b64, connect, frames_until, next_frame, send, type_of, + DEADLINE, FrameLines, assert_error, b64, connect_with, expect_frame, frames_until, next_frame, + send, type_of, }; async fn types_until(lines: &mut FrameLines, terminal: &str) -> Vec { @@ -25,14 +26,12 @@ async fn types_until(lines: &mut FrameLines, terminal: &str) -> Vec { #[tokio::test] async fn rpcs_run_back_to_back_on_one_connection() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - - send(&mut cw, json!({"v":1,"op":"info"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "info"); + let (mut cw, mut lines, handle) = connect_with(&state, json!({"v":1,"op":"info"})).await; + expect_frame(&mut lines, "info").await; send(&mut cw, json!({"v":1,"op":"fs_stat","path":"/"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "stat"); + expect_frame(&mut lines, "stat").await; send(&mut cw, json!({"v":1,"op":"ps"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "procs"); + expect_frame(&mut lines, "procs").await; cw.shutdown().await.unwrap(); timeout(DEADLINE, handle) @@ -45,10 +44,8 @@ async fn rpcs_run_back_to_back_on_one_connection() { #[tokio::test] async fn exec_hands_the_connection_back_after_exit() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send( - &mut cw, + let (mut cw, mut lines, _) = connect_with( + &state, json!({"v":1,"op":"exec","argv":["echo","hi"],"detach":false}), ) .await; @@ -62,16 +59,14 @@ async fn exec_hands_the_connection_back_after_exit() { assert!(types.iter().any(|t| t == "stdout"), "got {types:?}"); send(&mut cw, json!({"v":1,"op":"info"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "info"); + expect_frame(&mut lines, "info").await; } #[tokio::test] async fn late_stdin_after_exit_is_dropped() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send( - &mut cw, + let (mut cw, mut lines, _) = connect_with( + &state, json!({"v":1,"op":"exec","argv":["true"],"detach":false}), ) .await; @@ -81,16 +76,14 @@ async fn late_stdin_after_exit_is_dropped() { send(&mut cw, json!({"v":1,"op":"stdin","data":b64(b"late")})).await; send(&mut cw, json!({"v":1,"op":"stdin_close"})).await; send(&mut cw, json!({"v":1,"op":"info"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "info"); + expect_frame(&mut lines, "info").await; } #[tokio::test] async fn a_malformed_request_after_exec_is_bad_request() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send( - &mut cw, + let (mut cw, mut lines, _) = connect_with( + &state, json!({"v":1,"op":"exec","argv":["true"],"detach":false}), ) .await; @@ -115,46 +108,39 @@ async fn upload_hands_the_connection_back_after_done() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("f").to_string_lossy().into_owned(); let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send(&mut cw, json!({"v":1,"op":"fs_write","path":path})).await; + let (mut cw, mut lines, _) = + connect_with(&state, json!({"v":1,"op":"fs_write","path":path})).await; send(&mut cw, json!({"v":1,"op":"data","data":b64(b"hello")})).await; send(&mut cw, json!({"v":1,"op":"data_end"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "done"); + expect_frame(&mut lines, "done").await; send(&mut cw, json!({"v":1,"op":"fs_stat","path":path})).await; - let stat = next_frame(&mut lines).await; - assert_eq!(type_of(&stat), "stat", "got {stat}"); + let stat = expect_frame(&mut lines, "stat").await; assert_eq!(stat["info"]["size"], 5, "got {stat}"); } #[tokio::test] async fn a_request_during_exec_ends_its_stdin_and_runs_after_it() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send( - &mut cw, + let (mut cw, mut lines, _) = connect_with( + &state, json!({"v":1,"op":"exec","argv":["cat"],"detach":false}), ) .await; - assert_eq!(type_of(&next_frame(&mut lines).await), "started"); + expect_frame(&mut lines, "started").await; send(&mut cw, json!({"v":1,"op":"info"})).await; - let exit = next_frame(&mut lines).await; - assert_eq!(type_of(&exit), "exit", "got {exit}"); + let exit = expect_frame(&mut lines, "exit").await; assert_eq!(exit["code"], 0, "got {exit}"); - assert_eq!(type_of(&next_frame(&mut lines).await), "info"); + expect_frame(&mut lines, "info").await; } #[tokio::test] async fn a_leading_input_frame_is_bad_request_and_the_connection_stays() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send(&mut cw, json!({"v":1,"op":"stdin_close"})).await; + let (mut cw, mut lines, _) = connect_with(&state, json!({"v":1,"op":"stdin_close"})).await; let err = next_frame(&mut lines).await; assert_error(&err, "bad_request"); send(&mut cw, json!({"v":1,"op":"info"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "info"); + expect_frame(&mut lines, "info").await; } diff --git a/silkd/tests/lsp_e2e.rs b/silkd/tests/lsp_e2e.rs index 6055cca09..590f5b882 100644 --- a/silkd/tests/lsp_e2e.rs +++ b/silkd/tests/lsp_e2e.rs @@ -12,7 +12,10 @@ use tempfile::TempDir; use tokio::io::AsyncWriteExt; use tokio::time::timeout; -use common::{DEADLINE, assert_error, b64, connect, decode, next_frame, one, send, type_of}; +use common::{ + DEADLINE, FrameLines, FrameWriter, alive, assert_error, b64, connect_with, decode, + expect_frame, one, send, type_of, under_deadline, +}; const FAKE_SERVER: &str = r#"#!/bin/sh IFS= read -r line @@ -51,6 +54,20 @@ async fn started_server(lang: &str, server_body: &str) -> (TempDir, Arc, (env, state, server_id) } +async fn lsp_stream( + state: &Arc, + server_id: &str, +) -> ( + FrameWriter, + FrameLines, + tokio::task::JoinHandle>, +) { + let (cw, mut out, handle) = + connect_with(state, json!({"op":"lsp_request","server_id":server_id})).await; + expect_frame(&mut out, "ready").await; + (cw, out, handle) +} + async fn assert_stopped(state: &Arc, server_id: &str) { let gone = lsp_stop(state, server_id).await; assert_eq!(gone.last().unwrap()["kind"], "not_found", "{gone:?}"); @@ -67,15 +84,12 @@ async fn lsp_stop(state: &Arc, server_id: &str) -> Vec { #[tokio::test] async fn lsp_start_missing_manifest_is_not_found() { let _env_lock = ENV_LOCK.lock().await; - let frames = timeout( - DEADLINE, - one( - &Arc::new(State::new()), - &json!({"op":"lsp_start","language":"nope"}).to_string(), - ), - ) - .await - .expect("deadline"); + let _env = manifest_env("#!/bin/sh\n"); + let frames = under_deadline(one( + &Arc::new(State::new()), + &json!({"op":"lsp_start","language":"nope"}).to_string(), + )) + .await; let last = frames.last().unwrap(); assert_error(last, "not_found"); } @@ -105,15 +119,12 @@ async fn lsp_broker_relays_to_the_server() { let _env_lock = ENV_LOCK.lock().await; let (_env, state, server_id) = started_server("faketest", FAKE_SERVER).await; - let (mut cw, mut out, handle) = connect(&state); - send(&mut cw, json!({"op":"lsp_request","server_id":server_id})).await; - assert_eq!(type_of(&next_frame(&mut out).await), "ready"); + let (mut cw, mut out, handle) = lsp_stream(&state, &server_id).await; send(&mut cw, json!({"op":"data","data":b64(b"ping\n")})).await; - let frame = next_frame(&mut out).await; - assert_eq!(type_of(&frame), "data"); + let frame = expect_frame(&mut out, "data").await; assert_eq!(decode(&frame), b"reply:ping\n"); - assert_eq!(type_of(&next_frame(&mut out).await), "done"); + expect_frame(&mut out, "done").await; cw.shutdown().await.unwrap(); handle.await.unwrap().unwrap(); @@ -129,6 +140,39 @@ async fn lsp_stop_kills_an_idle_server() { assert_stopped(&state, &server_id).await; } +#[tokio::test] +async fn lsp_stop_kills_the_servers_children_too() { + let _env_lock = ENV_LOCK.lock().await; + let (env, state, server_id) = started_server( + "grouptest", + "#!/bin/sh\nsleep 300 &\necho $! > \"$(dirname \"$0\")/child\"\nwait\n", + ) + .await; + let pid_file = env.path().join("child"); + let deadline = tokio::time::Instant::now() + DEADLINE; + let pid: u32 = loop { + if let Some(pid) = std::fs::read_to_string(&pid_file) + .ok() + .and_then(|s| s.trim().parse().ok()) + { + break pid; + } + assert!( + tokio::time::Instant::now() < deadline, + "server never forked" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + }; + lsp_stop(&state, &server_id).await; + while alive(pid) { + assert!( + tokio::time::Instant::now() < deadline, + "server child {pid} outlived lsp_stop" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } +} + #[tokio::test] async fn lsp_request_reaps_when_the_client_vanishes_before_ready() { let _env_lock = ENV_LOCK.lock().await; @@ -140,9 +184,7 @@ async fn lsp_request_reaps_when_the_client_vanishes_before_ready() { client.write_all(b"\n").await.unwrap(); drop(client); let (sr, sw) = tokio::io::split(server); - let served = timeout(DEADLINE, state.serve(tokio::io::BufReader::new(sr), sw)) - .await - .expect("deadline"); + let served = under_deadline(state.serve(tokio::io::BufReader::new(sr), sw)).await; assert!(served.is_err(), "the ready write must fail: {served:?}"); assert_stopped(&state, &server_id).await; @@ -154,16 +196,13 @@ async fn lsp_data_end_half_closes_stdin() { let (_env, state, server_id) = started_server("eoftest", "#!/bin/sh\nprintf 'ate:%s\\n' $(cat | wc -c)\n").await; - let (mut cw, mut out, handle) = connect(&state); - send(&mut cw, json!({"op":"lsp_request","server_id":server_id})).await; - assert_eq!(type_of(&next_frame(&mut out).await), "ready"); + let (mut cw, mut out, handle) = lsp_stream(&state, &server_id).await; send(&mut cw, json!({"op":"data","data":b64(b"12345")})).await; send(&mut cw, json!({"op":"data_end"})).await; - let frame = next_frame(&mut out).await; - assert_eq!(type_of(&frame), "data"); + let frame = expect_frame(&mut out, "data").await; assert_eq!(decode(&frame), b"ate:5\n"); - assert_eq!(type_of(&next_frame(&mut out).await), "done"); + expect_frame(&mut out, "done").await; cw.shutdown().await.unwrap(); handle.await.unwrap().unwrap(); } @@ -175,9 +214,7 @@ async fn lsp_client_hang_up_reaps_a_silent_server() { let (_env, state, server_id) = started_server("silenttest", "#!/bin/sh\nexec sleep 60\n").await; - let (mut cw, mut out, handle) = connect(&state); - send(&mut cw, json!({"op":"lsp_request","server_id":server_id})).await; - assert_eq!(type_of(&next_frame(&mut out).await), "ready"); + let (mut cw, out, handle) = lsp_stream(&state, &server_id).await; if half_closed_first { send(&mut cw, json!({"op":"data_end"})).await; } @@ -198,11 +235,9 @@ async fn lsp_request_during_a_stream_ends_it_and_runs_after_it() { let (_env, state, server_id) = started_server("pipelinetest", "#!/bin/sh\nexec sleep 60\n").await; - let (mut cw, mut out, _) = connect(&state); - send(&mut cw, json!({"op":"lsp_request","server_id":server_id})).await; - assert_eq!(type_of(&next_frame(&mut out).await), "ready"); + let (mut cw, mut out, _) = lsp_stream(&state, &server_id).await; send(&mut cw, json!({"op":"info"})).await; - assert_eq!(type_of(&next_frame(&mut out).await), "done"); - assert_eq!(type_of(&next_frame(&mut out).await), "info"); + expect_frame(&mut out, "done").await; + expect_frame(&mut out, "info").await; assert_stopped(&state, &server_id).await; } diff --git a/silkd/tests/pty_e2e.rs b/silkd/tests/pty_e2e.rs index 723a66f96..92f3d0155 100644 --- a/silkd/tests/pty_e2e.rs +++ b/silkd/tests/pty_e2e.rs @@ -18,7 +18,8 @@ use tokio::io::{AsyncWrite, AsyncWriteExt}; use tokio::sync::mpsc; use common::{ - FrameLines, assert_error, b64, connect, decode, frames_until, one, request, send, type_of, + FrameLines, FrameWriter, alive, assert_error, b64, connect_with, decode, frames_until, one, + proc_entry, request, send, type_of, wait_for_path, }; struct FailingSink(Vec); @@ -46,10 +47,19 @@ async fn read_until(lines: &mut FrameLines, pred: impl Fn(&Value) -> bool) -> Va frames_until(lines, pred).await.pop().unwrap() } +async fn open_pty( + state: &Arc, +) -> ( + FrameWriter, + FrameLines, + tokio::task::JoinHandle>, +) { + connect_with(state, json!({"op":"pty_open","cols":80,"rows":24})).await +} + async fn wait_gone(pid: u32) { let deadline = tokio::time::Instant::now() + Duration::from_secs(5); - // SAFETY: kill(2) with signal 0 only probes whether the pid exists; it takes no pointers. - while unsafe { libc::kill(pid as libc::pid_t, 0) } == 0 { + while alive(pid) { assert!( tokio::time::Instant::now() < deadline, "shell {pid} still alive 5 s after the started write failed" @@ -61,8 +71,7 @@ async fn wait_gone(pid: u32) { #[tokio::test] async fn pty_runs_a_shell_and_echoes() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; + let (mut cw, mut lines, handle) = open_pty(&state).await; let started = read_until(&mut lines, |v| v["type"] == "started").await; let pid = started["pid"].as_u64().unwrap(); @@ -90,8 +99,7 @@ async fn pty_runs_a_shell_and_echoes() { #[tokio::test] async fn the_pty_shell_holds_no_master_fd() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; + let (mut cw, mut lines, handle) = open_pty(&state).await; read_until(&mut lines, |v| v["type"] == "started").await; send( &mut cw, @@ -113,21 +121,13 @@ async fn the_pty_shell_holds_no_master_fd() { #[tokio::test] async fn pty_appears_in_ps_and_resizes() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; + let (cw, mut lines, _) = open_pty(&state).await; let pid = read_until(&mut lines, |v| v["type"] == "started").await["pid"] .as_u64() .unwrap(); let ps = one(&state, r#"{"op":"ps"}"#).await; - assert!( - ps[0]["procs"] - .as_array() - .unwrap() - .iter() - .any(|p| p["pid"].as_u64() == Some(pid)), - "pty not in ps: {ps:?}" - ); + assert!(proc_entry(&ps, pid).is_some(), "pty not in ps: {ps:?}"); let resized = one( &state, &json!({"op":"pty_resize","pid":pid,"cols":120,"rows":40}).to_string(), @@ -147,8 +147,7 @@ async fn resize_unknown_pid_is_not_found() { #[tokio::test] async fn stdin_nobody_reads_after_the_shell_exits_does_not_wedge_the_next_request() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; + let (mut cw, mut lines, handle) = open_pty(&state).await; read_until(&mut lines, |v| v["type"] == "started").await; send( &mut cw, @@ -178,8 +177,7 @@ async fn stdin_nobody_reads_after_the_shell_exits_does_not_wedge_the_next_reques #[tokio::test] async fn pty_disconnect_tears_down_without_spin() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; + let (cw, mut lines, handle) = open_pty(&state).await; let pid = read_until(&mut lines, |v| v["type"] == "started").await["pid"] .as_u64() .unwrap(); @@ -193,15 +191,42 @@ async fn pty_disconnect_tears_down_without_spin() { .expect("pty did not tear down on disconnect"); let ps = one(&state, r#"{"op":"ps"}"#).await; assert!( - !ps[0]["procs"] - .as_array() - .unwrap() - .iter() - .any(|p| p["pid"].as_u64() == Some(pid)), + proc_entry(&ps, pid).is_none(), "pty still in table after disconnect: {ps:?}" ); } +#[tokio::test] +async fn pty_disconnect_hangs_up_background_jobs() { + let dir = tempfile::tempdir().unwrap(); + let pid_file = dir.path().join("job"); + let state = Arc::new(State::new()); + let (mut cw, mut lines, handle) = open_pty(&state).await; + read_until(&mut lines, |v| v["type"] == "started").await; + let cmd = format!("sleep 300 & echo $! > {}\n", pid_file.display()); + send(&mut cw, json!({"op":"stdin","data":b64(cmd.as_bytes())})).await; + wait_for_path(&pid_file).await; + let job: u32 = loop { + if let Ok(pid) = std::fs::read_to_string(&pid_file).unwrap().trim().parse() { + break pid; + } + tokio::time::sleep(Duration::from_millis(10)).await; + }; + drop(lines); + drop(cw); + let _ = tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("pty did not tear down on disconnect"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while alive(job) { + assert!( + tokio::time::Instant::now() < deadline, + "background job {job} outlived the hang-up" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + #[tokio::test] async fn pty_started_write_failure_kills_the_shell_and_unregisters_it() { let table = Table::default(); diff --git a/silkd/tests/session_e2e.rs b/silkd/tests/session_e2e.rs index 5235e0c43..798a67c90 100644 --- a/silkd/tests/session_e2e.rs +++ b/silkd/tests/session_e2e.rs @@ -10,7 +10,7 @@ use serde_json::{Value, json}; use silkd::server::State; use tokio::task::JoinHandle; -use common::{assert_error, one, stdout_body, type_of}; +use common::{assert_error, one, stdout_body, type_of, wait_for_path}; async fn create(state: &Arc) -> String { create_with(state, json!({"op":"session_create"})).await @@ -31,10 +31,28 @@ async fn sh(state: &Arc, id: &str, cmd: &[&str]) -> Vec { .await } -fn spawn_sh(state: &Arc, id: &str, cmd: &'static [&'static str]) -> JoinHandle> { +async fn spawn_busy(state: &Arc, id: &str, seconds: &'static str) -> JoinHandle> { + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("started"); + let marker_arg = marker.to_str().unwrap().to_string(); let s = Arc::clone(state); let sid = id.to_string(); - tokio::spawn(async move { sh(&s, &sid, cmd).await }) + let busy = tokio::spawn(async move { + sh( + &s, + &sid, + &[ + "sh", + "-c", + "touch \"$0\"; exec sleep \"$1\"", + &marker_arg, + seconds, + ], + ) + .await + }); + wait_for_path(&marker).await; + busy } #[tokio::test] @@ -216,8 +234,7 @@ async fn reap_idle_removes_idle_sessions() { async fn reap_skips_a_session_running_a_command() { let state = Arc::new(State::new()); let id = create(&state).await; - let busy = spawn_sh(&state, &id, &["sleep", "1"]); - tokio::time::sleep(Duration::from_millis(150)).await; + let busy = spawn_busy(&state, &id, "1").await; let reaped = state.sessions.reap_idle(Duration::ZERO); assert_eq!(reaped, 0, "a running session must not be reaped"); let done = busy.await.unwrap(); @@ -242,8 +259,7 @@ async fn session_argv_with_metacharacters_is_one_literal_token() { async fn session_rm_unwedges_a_running_external_command() { let state = Arc::new(State::new()); let id = create(&state).await; - let busy = spawn_sh(&state, &id, &["sleep", "3600"]); - tokio::time::sleep(Duration::from_millis(200)).await; + let busy = spawn_busy(&state, &id, "3600").await; let rm = one(&state, &json!({"op":"session_rm","id":id}).to_string()).await; assert_eq!(type_of(&rm[0]), "done"); let done = tokio::time::timeout(Duration::from_secs(5), busy) @@ -308,8 +324,7 @@ async fn a_dead_session_command_leaves_a_recreated_session_alone() { ) .await; assert_eq!(type_of(bg.last().unwrap()), "exit", "{bg:?}"); - let busy = spawn_sh(&state, &id, &["sleep", "3600"]); - tokio::time::sleep(Duration::from_millis(200)).await; + let busy = spawn_busy(&state, &id, "3600").await; let rm = one(&state, &json!({"op":"session_rm","id":id}).to_string()).await; assert_eq!(type_of(&rm[0]), "done"); let again = create_with(&state, json!({"op":"session_create","id":"s"})).await; diff --git a/silkd/tests/tree_e2e.rs b/silkd/tests/tree_e2e.rs index 41ae0067d..b1a168e27 100644 --- a/silkd/tests/tree_e2e.rs +++ b/silkd/tests/tree_e2e.rs @@ -10,7 +10,7 @@ use std::process::Command; use serde_json::json; -use common::{assert_error, b64, data_frames, exchange, payload, request, type_of}; +use common::{assert_error, b64, exchange, fs_push, payload, request, type_of}; fn sys_tar_create(dir: &Path) -> Vec { let out = Command::new("tar") @@ -46,9 +46,7 @@ async fn push_extracts_a_tar_stream() { let dest = tempfile::tempdir().unwrap(); let dest_path = dest.path().join("proj"); - let mut lines = vec![json!({"op":"fs_push","dest":dest_path.to_str().unwrap()}).to_string()]; - lines.extend(data_frames(&archive)); - let frames = exchange(&lines).await; + let frames = fs_push(&dest_path, &archive).await; assert_eq!( type_of(&frames[0]), @@ -92,9 +90,7 @@ async fn push_then_pull_roundtrips_a_tree() { let work = tempfile::tempdir().unwrap(); let dest = work.path().join("t"); - let mut lines = vec![json!({"op":"fs_push","dest":dest.to_str().unwrap()}).to_string()]; - lines.extend(data_frames(&archive)); - assert_eq!(type_of(&exchange(&lines).await[0]), "done"); + assert_eq!(type_of(&fs_push(&dest, &archive).await[0]), "done"); let pulled = request(json!({"op":"fs_pull","path":dest.to_str().unwrap()})).await; let out_archive = payload(&pulled, "data"); @@ -238,9 +234,7 @@ async fn push_overlays_existing_tree_and_cleans_staging() { std::fs::create_dir(dest.path().join("sub")).unwrap(); std::fs::write(dest.path().join("sub/old.txt"), b"OLD").unwrap(); - let mut lines = vec![json!({"op":"fs_push","dest":dest.path().to_str().unwrap()}).to_string()]; - lines.extend(data_frames(&archive)); - let frames = exchange(&lines).await; + let frames = fs_push(dest.path(), &archive).await; assert_eq!(type_of(&frames[0]), "done", "push failed: {frames:?}"); assert_eq!( diff --git a/silkd/tests/tun_e2e.rs b/silkd/tests/tun_e2e.rs index b1296a216..71d2b45ec 100644 --- a/silkd/tests/tun_e2e.rs +++ b/silkd/tests/tun_e2e.rs @@ -2,6 +2,7 @@ #![cfg(target_os = "linux")] #![allow(clippy::unwrap_used, clippy::expect_used)] +use std::io::{Read, Write}; use std::net::{Ipv4Addr, SocketAddr}; use std::os::fd::{AsFd, AsRawFd}; use std::os::unix::net::UnixStream; @@ -562,8 +563,6 @@ fn forward( to: Ipv4Addr, ) -> JoinHandle<()> { std::thread::spawn(move || { - use std::io::{Read, Write}; - let mut r = std::io::BufReader::with_capacity(256 << 10, r); let mut hello = [0u8; HELLO_LEN]; if r.read_exact(&mut hello).is_err() || w.write_all(&hello).is_err() { diff --git a/silkd/tests/watch_e2e.rs b/silkd/tests/watch_e2e.rs index 9bd5d5585..bbb77d0fd 100644 --- a/silkd/tests/watch_e2e.rs +++ b/silkd/tests/watch_e2e.rs @@ -8,19 +8,18 @@ use std::time::Duration; use serde_json::json; use silkd::server::State; -use common::{assert_error, connect, next_frame, request, send, type_of}; +use common::{assert_error, connect_with, expect_frame, next_frame, request, type_of}; #[tokio::test] async fn watch_streams_events_until_disconnect() { let dir = tempfile::tempdir().unwrap(); let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send( - &mut cw, + let (cw, mut lines, handle) = connect_with( + &state, json!({"op": "fs_watch", "path": dir.path().to_str().unwrap(), "recursive": true}), ) .await; - assert_eq!(type_of(&next_frame(&mut lines).await), "ready"); + expect_frame(&mut lines, "ready").await; tokio::fs::write(dir.path().join("new.txt"), b"hi") .await .unwrap();