diff --git a/client/Chart.yaml b/client/Chart.yaml index a0485a84..4b8b89cd 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.95 -appVersion: "1.9.95" +version: 1.9.96 +appVersion: "1.9.96" keywords: - tracebloc - kubernetes diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 83c70d98..b5e1705a 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -663,7 +663,8 @@ spec: {{- if and (default dict .Values.egressProxy).enabled (default dict .Values.egressProxy).routeWorkloads }} # client-runtime#102: when routing is enabled, jobs-manager injects # HTTPS_PROXY= into training pods (and suppresses raw HTTP_PROXY_HOST). - # Inert until egressProxy.routeWorkloads=true. + # routeWorkloads defaults true (RFC-0003 D6), so this renders by default; + # set egressProxy.routeWorkloads=false to opt a fleet out of gateway routing. - name: EGRESS_PROXY_URL value: "http://egress-proxy-service:{{ (default dict .Values.egressProxy).port | default 3128 }}" {{- end }} diff --git a/client/templates/network-policy-training.yaml b/client/templates/network-policy-training.yaml index 08b350da..73d62264 100644 --- a/client/templates/network-policy-training.yaml +++ b/client/templates/network-policy-training.yaml @@ -61,11 +61,12 @@ spec: protocol: UDP - port: 53 protocol: TCP - {{- /* Rule 2 (external HTTPS) is the egress hole SECURITY §8.2 closes. Gated on - networkPolicy.training.allowExternalHttps via `dig` with a default of TRUE: - an absent key (helm upgrade --reuse-values from a release predating it) - keeps the rule — old behaviour — so only an explicit `false` drops it, once - an operator has verified the egress gateway on that cluster (#102). */}} + {{- /* Rule 2 (external HTTPS) is the egress hole SECURITY §8.2 closes. The + shipped default is now `false` (values.yaml — RFC-0003 D6 deny-by-default), + so a fresh render drops this rule. The `dig` default of TRUE here is the + legacy fallback ONLY: an absent key (helm upgrade --reuse-values from a + release predating the key) keeps the rule — old permissive behaviour — and + an operator can still opt a fleet back out with an explicit `true` (#102). */}} {{- if dig "allowExternalHttps" true .Values.networkPolicy.training }} # 2. External HTTPS — everything NOT in the cluster's pod/service CIDRs. # Training pods reach the backend / Azure Service Bus / App Insights directly. diff --git a/client/tests/egress_enforcement_check_test.yaml b/client/tests/egress_enforcement_check_test.yaml index 34989e76..8a1d3dad 100644 --- a/client/tests/egress_enforcement_check_test.yaml +++ b/client/tests/egress_enforcement_check_test.yaml @@ -9,7 +9,21 @@ set: clientId: "test-id" clientPassword: "test" tests: - - it: does NOT render by default (lockdown off — allowExternalHttps defaults true) + - it: renders by default (lockdown on — allowExternalHttps defaults false) + asserts: + - hasDocuments: + count: 1 + + # Opt-out coverage: with lockdown now the default, an operator who sets + # allowExternalHttps=true has re-opened direct egress, so the seal-check has + # nothing to verify and must NOT render. This is the assertion the inverted + # default-render case above used to carry (Bugbot on this PR) — without it a + # regression that still emits the Job after opt-out would stay green. + - it: does NOT render when an operator opts back out (allowExternalHttps=true) + set: + networkPolicy: + training: + allowExternalHttps: true asserts: - hasDocuments: count: 0 diff --git a/client/tests/egress_proxy_test.yaml b/client/tests/egress_proxy_test.yaml index b0a972a4..6887caa5 100644 --- a/client/tests/egress_proxy_test.yaml +++ b/client/tests/egress_proxy_test.yaml @@ -152,12 +152,9 @@ tests: # --- Step 1: routing training pods through the gateway --- - - it: jobs-manager gets EGRESS_PROXY_URL only when routeWorkloads is enabled + - it: jobs-manager gets EGRESS_PROXY_URL by default (routeWorkloads defaults true) template: templates/jobs-manager-deployment.yaml documentIndex: 0 - set: - egressProxy: - routeWorkloads: true asserts: - contains: path: spec.template.spec.containers[0].env @@ -165,9 +162,12 @@ tests: name: EGRESS_PROXY_URL value: "http://egress-proxy-service:3128" - - it: jobs-manager does NOT get EGRESS_PROXY_URL by default (routeWorkloads false) + - it: jobs-manager omits EGRESS_PROXY_URL when routeWorkloads is disabled template: templates/jobs-manager-deployment.yaml documentIndex: 0 + set: + egressProxy: + routeWorkloads: false asserts: - notContains: path: spec.template.spec.containers[0].env @@ -197,10 +197,10 @@ tests: # --- Step 3: the lockdown — drop the external 0.0.0.0/0:443 rule --- - - it: keeps the external 443 egress rule by default (allowExternalHttps true) + - it: drops the external 443 egress rule by default (allowExternalHttps defaults false) template: templates/network-policy-training.yaml asserts: - - contains: + - notContains: path: spec.egress content: to: @@ -214,14 +214,14 @@ tests: - port: 443 protocol: TCP - - it: drops the external 443 rule but keeps the gateway path when allowExternalHttps is false + - it: renders the external 443 rule and keeps the gateway path when allowExternalHttps is opted back to true template: templates/network-policy-training.yaml set: networkPolicy: training: - allowExternalHttps: false + allowExternalHttps: true asserts: - - notContains: + - contains: path: spec.egress content: to: diff --git a/client/tests/network_policy_test.yaml b/client/tests/network_policy_test.yaml index dfc19893..84367dbb 100644 --- a/client/tests/network_policy_test.yaml +++ b/client/tests/network_policy_test.yaml @@ -156,7 +156,7 @@ tests: networkPolicy: training: enabled: true - allowExternalHttps: true # pin the default — this asserts the rule-2 hole + allowExternalHttps: true # opt back out (default is now false) — asserts the rule-2 hole still renders on true dnsNamespace: kube-system dnsSelector: k8s-app: kube-dns @@ -266,8 +266,8 @@ tests: port: 8888 protocol: TCP # locked-down egress is exactly 4 rules: DNS + MySQL + requests-proxy + - # egress-proxy. NOTE: `egressProxy.enabled` DEFAULTS TRUE (values.yaml — - # only `routeWorkloads` defaults false), so the gateway's permit rule + # egress-proxy. NOTE: `egressProxy.enabled` and `routeWorkloads` both + # DEFAULT TRUE (values.yaml — RFC-0003 D6), so the gateway's permit rule # renders here even though this case doesn't set it. The direct length # assertion makes the shape unmissable and catches a stray appended rule # (Saqlain review on #537). diff --git a/client/values.schema.json b/client/values.schema.json index 3190f808..44d14b65 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -414,8 +414,8 @@ }, "allowExternalHttps": { "type": "boolean", - "default": true, - "description": "When false, drop the 0.0.0.0/0:443 egress rule so training pods reach only DNS, MySQL, requests-proxy and the egress gateway (SECURITY \u00a78.2 / client-runtime#102). Default true keeps existing behaviour; flip per-fleet after verifying the egress gateway works (G2)." + "default": false, + "description": "When false (the default, RFC-0003 D6), drop the 0.0.0.0/0:443 egress rule so training pods reach only DNS, MySQL, requests-proxy and the egress gateway (SECURITY \u00a78.2 / client-runtime#102). Set true to opt a fleet back out to direct egress. An absent key on a `--reuse-values` upgrade from a release predating it keeps the old permissive behaviour." }, "enforcementProbeHost": { "type": "string", @@ -683,8 +683,8 @@ }, "routeWorkloads": { "type": "boolean", - "default": false, - "description": "Route training-pod outbound HTTPS through the gateway (jobs-manager injects HTTPS_PROXY). Default false \u2014 enable per-fleet, verify a run, then drop the direct egress rule (networkPolicy.training.allowExternalHttps=false)." + "default": true, + "description": "Route training-pod outbound HTTPS through the gateway (jobs-manager injects HTTPS_PROXY). Default true (RFC-0003 D6, deny-by-default); set false to opt a fleet out of gateway routing." }, "port": { "type": "integer", diff --git a/client/values.yaml b/client/values.yaml index 9498f355..a12e5606 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -370,11 +370,12 @@ networkPolicy: # Egress lockdown (SECURITY §8.2 / client-runtime#102). When false, the # training NetworkPolicy DROPS the 0.0.0.0/0:443 rule, so training pods can # reach only DNS, in-cluster MySQL, the requests-proxy, and the egress - # gateway. Default true keeps the fleet unchanged; flip OFF per-fleet AFTER - # verifying the egress gateway works on that cluster (G2). The template + # gateway. Shipped default is now FALSE — deny-by-default egress (RFC-0003 D6 / + # client-runtime#199): new installs are locked down, external HTTPS goes through + # the gateway allowlist. Set true only to opt a fleet back out. The template # nil-guards this key, so a `helm upgrade --reuse-values` from a release # predating it keeps the old behaviour (rule present). - allowExternalHttps: true + allowExternalHttps: false # client-runtime#104: enforcement check, run via `helm test ` after # flipping the lockdown (allowExternalHttps=false). A training-labelled pod opens a # direct TCP connection to this host's :443 and RETRIES until it is blocked (PASS) or @@ -480,10 +481,11 @@ sealCheck: # so the training NetworkPolicy never selects it (it keeps its own egress). egressProxy: enabled: true - # Route training-pod outbound HTTPS through the gateway (Step 1 of #102). Default - # FALSE so the gateway ships inert; flip per-fleet to true, verify a real training - # run, THEN set networkPolicy.training.allowExternalHttps=false to drop direct egress. - routeWorkloads: false + # Route training-pod outbound HTTPS through the gateway (Step 1 of #102). Shipped + # default is now TRUE — deny-by-default egress (RFC-0003 D6 / client-runtime#199): + # training egress flows through the gateway allowlist, paired with + # networkPolicy.training.allowExternalHttps=false. Set false only to opt back out. + routeWorkloads: true # squid image, pinned by multi-arch (amd64+arm64) index digest — tracebloc pins # all images by digest. `tag` stays for readability; the digest is authoritative. # ubuntu/squid:6.6-24.04_beta (Ubuntu 24.04 LTS base), resolved 2026-06-10. @@ -594,8 +596,9 @@ egressProxy: # DaemonSet on every customer node spooling to disk with nothing it can deliver — # filling toward the 1 GiB cap per node for no benefit, which is precisely the # "telemetry must never be the reason a node fills" risk D7 exists to bound. -# Same posture as `egressProxy.routeWorkloads`: the component ships inert and is -# flipped per fleet once its prerequisite is in place. +# Ships inert and is flipped per fleet once its prerequisite is in place — the +# posture `egressProxy.routeWorkloads` had before RFC-0003 D6 flipped it on by +# default. # # CLASS A ONLY (D12). The receiver is scoped to the control-plane containers this # chart owns; training and ingestion pods are Class B and stay out until diff --git a/docs/SEAL-CHECK.md b/docs/SEAL-CHECK.md index 68db3a39..f82d5192 100644 --- a/docs/SEAL-CHECK.md +++ b/docs/SEAL-CHECK.md @@ -81,9 +81,9 @@ these): (the storage check's ServiceAccount/RBAC) deliberately do not — counting them would inflate the suite. - A check that does not *render* (its gating values turned it off, or its - preconditions are not declared — e.g. the egress-enforcement probe before - the lockdown is flipped) is **not part of the suite on that cluster**, and - the values that gated it away say why. + preconditions are not declared — e.g. the egress-enforcement probe when an + operator has opted a fleet back out with `allowExternalHttps=true`) is **not + part of the suite on that cluster**, and the values that gated it away say why. - Log lines are human-oriented and not part of the contract; the machine contract today is *labels + Job exit status*. (A structured verdict is the CLI's job — cli#393.) @@ -155,7 +155,7 @@ on the live cluster when the corresponding check runs.* | Guarantee | k3d local (k3s) | EKS | AKS | OpenShift | bare metal | |---|---|---|---|---|---| -| Training egress blocked (NetworkPolicy) | **Substrate verified; full-probe run pending** — k3s enforces egress NetworkPolicy (k3d v5.8.3 / k3s v1.33.6+k3s1, 2026-07-30; see §8.4 Status), full-chart `egress-enforcement` probe run not yet recorded | **Substrate verified; full-probe run pending** — dev fleet `tb-client-dev-templates` runs the VPC CNI netpol agent (v1.2.7, `--enable-network-policy=true`, mode `standard`, 2026-08-24; see EKS Status below), full-chart `egress-enforcement` probe not recorded (per-fleet lockdown held — client-runtime#199). Other EKS CNIs (Calico / Cilium) — **verified** by `egress-enforcement` once the lockdown is flipped | Conditional on CNI (Azure NPM / Calico) — **verified** by `egress-enforcement` once the lockdown is flipped | OVN-Kubernetes enforces by default — still **verified** by `egress-enforcement` | Conditional on CNI (Flannel alone does not enforce) — **verified** by `egress-enforcement` | +| Training egress blocked (NetworkPolicy) | **Substrate verified; full-probe run pending** — k3s enforces egress NetworkPolicy (k3d v5.8.3 / k3s v1.33.6+k3s1, 2026-07-30; see §8.4 Status), full-chart `egress-enforcement` probe run not yet recorded | **Substrate verified; full-probe run pending** — dev fleet `tb-client-dev-templates` runs the VPC CNI netpol agent (v1.2.7, `--enable-network-policy=true`, mode `standard`, 2026-08-24; see EKS Status below), full-chart `egress-enforcement` probe run recording still in progress across fleets (client-runtime#199; deny-by-default is the chart default as of 1.9.96). Other EKS CNIs (Calico / Cilium) — **verified** by `egress-enforcement` (renders by default as of 1.9.96; opt-out with `allowExternalHttps=true`) | Conditional on CNI (Azure NPM / Calico) — **verified** by `egress-enforcement` (renders by default as of 1.9.96; opt-out with `allowExternalHttps=true`) | OVN-Kubernetes enforces by default — still **verified** by `egress-enforcement` | Conditional on CNI (Flannel alone does not enforce) — **verified** by `egress-enforcement` | | Backend reachability (required egress) | **Verified** by `backend-reachability` | **Verified** | **Verified** | **Verified** | **Verified** | | Storage on the declared class, bound | **Verified** by `storage-assertions` | **Verified** | **Verified** | **Verified** (PV scan degraded if `clusterScope=false`) | **Verified** | | No unmanaged hostPath backing (dynamic mode) | **Verified** once the Option C flip lands (today's installer still declares hostPath mode → sub-check SKIPs, honestly) | **Verified** | **Verified** | **Verified** with `clusterScope=true`; partial (name check + explicit WARNING) otherwise | n/a — hostPath *is* the declared model (SKIP) | @@ -163,11 +163,14 @@ on the live cluster when the corresponding check runs.* Two lockdown caveats the suite states rather than hides: -- `egress-enforcement` only *renders* after the per-fleet lockdown flip - (`allowExternalHttps=false` — the RFC-0003 §8.1 rollout). Until that flip, - training-pod outbound :443 is deliberately open and there is no - enforcement to verify — the environment is **not sealed for egress** and - nothing here claims it is. +- `egress-enforcement` renders **by default** as of chart 1.9.96 + (`allowExternalHttps=false` is the shipped default — RFC-0003 D6), so a fresh + install seals training-pod outbound :443 and `helm test` runs this check. An + operator who opts a fleet back out (`allowExternalHttps=true`) re-opens direct + :443; the hook then does not render and there is no enforcement to verify — + that fleet is **not sealed for egress** until the lockdown is restored. (Charts + `≥ 1.7.0` and `< 1.9.96` shipped permissive, so on those the hook renders only + after an explicit flip.) - A rendered check that fails means the environment is **unsealed** for that guarantee until fixed — e.g. a CNI that does not enforce NetworkPolicy fails `egress-enforcement` with remediation hints, exactly so the lockdown diff --git a/docs/SECURITY.md b/docs/SECURITY.md index bc01059a..d7af302a 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -195,7 +195,7 @@ spec: - DNS lookups (needed to resolve backend + Azure endpoints) - In-cluster egress to MySQL (3306), the requests-proxy (8888), and the egress gateway (3128) -- Outbound HTTPS/443 to the public internet — **only while `networkPolicy.training.allowExternalHttps: true` (the current default).** Set it to `false` and this rule is dropped, so training pods reach external services only through the in-cluster egress gateway (see §8.2). +- Outbound HTTPS/443 to the public internet — **dropped by default as of 1.9.96 (`networkPolicy.training.allowExternalHttps: false`, deny-by-default).** Training pods reach external services only through the in-cluster egress gateway; set `allowExternalHttps: true` to opt a fleet back out (see §8.2). **Enforcement prerequisite:** every bullet above is a *request* to the CNI, not a guarantee. It holds only on a CNI that enforces **egress** NetworkPolicy — Calico, Cilium, OpenShift OVN-Kubernetes, Azure CNI created with a network policy, or the **EKS VPC CNI managed add-on with `enableNetworkPolicy=true`**. See [§5.1](#enforcing-cnis) for the full list and the EKS caveat, and §6.2 for how to verify it on a given cluster. On a non-enforcing CNI the policy object exists and blocks nothing. @@ -348,7 +348,9 @@ Anything else — notably **Flannel alone** and a **self-managed AWS VPC CNI** **Silent-no-enforcement risk:** If `networkPolicy.training.enabled: true` on a cluster whose CNI does not enforce, the policy is created but ignored. Customers must verify their CNI enforces NetworkPolicy before relying on this layer. We default the EKS `ci/eks-values.yaml` to `enabled: false` for this reason. -> Verified 2026-06 on `tb-client-dev-templates` (EKS, **self-managed** VPC CNI, NetworkPolicy disabled): a DNS-only egress NetworkPolicy did **not** block `https://example.com` — the probe returned `200`. On such a fleet, flipping `allowExternalHttps=false` is **cosmetic**: the rule renders and nothing enforces it. +> Verified 2026-06 on `tb-client-dev-templates` (then EKS **self-managed** VPC CNI, NetworkPolicy disabled): a DNS-only egress NetworkPolicy did **not** block `https://example.com` — the probe returned `200`. On such a fleet, flipping `allowExternalHttps=false` is **cosmetic**: the rule renders and nothing enforces it. +> +> **Updated 2026-08 — both tracebloc EKS fleets now enforce.** `tb-client-dev-templates` (dev/staging) and `tracebloc-clients-prod` (prod) have since moved to the **managed** `vpc-cni` add-on with `enableNetworkPolicy=true` (verified via `aws eks describe-addon --cluster-name --addon-name vpc-cni --query addon.configurationValues` → `{"enableNetworkPolicy":"true"}`). Egress NetworkPolicy is now **enforced** on both — the "cosmetic" caveat above applied to the older self-managed CNI, not to the current state. (Standard-mode reconcile window still applies; the `egress-enforcement` seal check retries across it.) ### 5.2 Pod Security Admission @@ -643,9 +645,9 @@ Known gaps between the current state and a fully-hardened setup, with the owner **Interim mitigation:** with the §8.2 egress lockdown enabled (`networkPolicy.training.allowExternalHttps: false`), a training pod can no longer reach Azure Service Bus directly — SB traffic goes through the in-cluster requests-proxy (which holds the connection strings), and the conn-strings are no longer injected into the pod. Until a fleet enables the lockdown the NetworkPolicy still allows direct outbound HTTPS. The scoped/short-TTL SAS-token plan above remains the durable fix. See §8.2. -### 8.2 Training-pod outbound HTTPS (G2) — **mechanism shipped (1.7.0), gated rollout** +### 8.2 Training-pod outbound HTTPS (G2) — **deny-by-default shipped (1.9.96); per-fleet enforcement still gated** -By default the NetworkPolicy still allows outbound HTTPS/443 so training pods can reach the backend, Azure Service Bus, and App Insights — so a malicious pod can still `requests.post()` to an arbitrary endpoint until the lockdown is enabled. +As of chart **1.9.96** the shipped defaults are deny-by-default (`egressProxy.routeWorkloads: true` + `networkPolicy.training.allowExternalHttps: false`, RFC-0003 D6 / client-runtime#199): a new or upgraded install routes training egress through the gateway allowlist and drops the direct external-443 rule, so a malicious pod can no longer `requests.post()` to an arbitrary endpoint. **This is a config default, not a guarantee** — it only *enforces* on a CNI that enforces egress NetworkPolicy (§5.1); on a non-enforcing fleet the rule renders and blocks nothing. Existing fleets and non-enforcing CNIs still require the per-fleet gated rollout below. Charts `≥ 1.7.0` and `< 1.9.96` shipped the mechanism permissive (`routeWorkloads: false`, `allowExternalHttps: true`). **Mechanism (chart 1.7.0, client-runtime#102):** an in-cluster **egress gateway** (`egressProxy` — a squid forward proxy) permits HTTPS CONNECT only to an FQDN allowlist (backend + App Insights) and chains to a corporate proxy via `cache_peer`. With routing on, jobs-manager injects `HTTPS_PROXY=egress-proxy-service:3128` into each training pod (and drops the raw `HTTP_PROXY_HOST`), so backend + App-Insights traffic flows through the gateway; Service Bus already goes via the requests-proxy. The pod then needs no direct internet, and the external-443 rule can be dropped. @@ -653,7 +655,7 @@ By default the NetworkPolicy still allows outbound HTTPS/443 so training pods ca **Gate 0 — CNI enforcement pre-flight (required, before anything else).** Run the [§6.2 egress pre-flight probe](#egress-preflight-probe) on this fleet. It must fail to connect. If it returns `200`, this fleet's CNI does **not** enforce egress and steps 1–3 buy nothing: the rule renders, nothing blocks, and the fleet reads as locked down when it is not. Fix the CNI first (§5.1 — on EKS, that usually means moving to the `vpc-cni` managed add-on with `enableNetworkPolicy=true`). Do not proceed on a fleet that fails this gate. -1. Upgrade to ≥ 1.7.0 — the gateway deploys, inert (`egressProxy.routeWorkloads: false`). +1. Upgrade to ≥ 1.9.96 — the gateway deploys with routing **on** by default (`egressProxy.routeWorkloads: true`). (On `≥ 1.7.0` and `< 1.9.96` it deploys inert; set `routeWorkloads: true` explicitly there.) 2. Set `egressProxy.routeWorkloads: true`; verify a training run completes via the gateway. 3. **Drain first.** Wait for in-flight experiments to finish (`kubectl -n get pods -l tracebloc.io/workload=training`). Step 4 changes the policy for *running* pods too, so a training pod mid-run that still reaches the internet directly — one whose image or user code has not picked up `HTTPS_PROXY` — fails at the moment of the flip rather than at submit time. Select **pods, not Jobs**: jobs-manager sets `tracebloc.io/workload: training` on the pod template only (`job.yaml`, `jobs_manager._prepare_job_config`), never on the Job object — which is all the NetworkPolicy needs, since its `podSelector` matches the pod. `get jobs -l …` therefore returns nothing even mid-run, which reads as a false all-clear. 4. Set `networkPolicy.training.allowExternalHttps: false` to drop the external-443 rule. @@ -805,6 +807,7 @@ Cross-reference for reviewers and contributors. - **2026-04** — Initial version. Documents the training-pod sandbox as shipped in client chart ≥ 1.0.4 and client-runtime images built from `develop` at that date. Reflects the narrow threat model (trusted platform, untrusted external data scientist submissions). - **2026-08** — Documented the MySQL database identity model (§4.1.1) and the `edgeuser` root-equivalence retirement (§8.10) — RFC-0003 D10 close-out, backend#1528. +- **2026-08** — Flipped the shipped chart defaults to **deny-by-default egress** (`egressProxy.routeWorkloads: true` + `networkPolicy.training.allowExternalHttps: false`, chart 1.9.96) so new/upgraded installs are locked down out of the box (RFC-0003 D6 / client-runtime#199). Per-fleet enforcement still follows the §8.2 gated rollout (CNI pre-flight → verify a run → drop the rule → seal-check); non-enforcing CNIs render the rule without blocking. - **2026-08** — Sharpened the §8.2 egress-lockdown guidance (tracebloc/client#248): named the enforcing CNIs concretely in §5.1 (including the EKS `vpc-cni` managed add-on `enableNetworkPolicy=true` prerequisite), added the §6.2 egress pre-flight probe as a required gate before the flip, and gave §8.2 a drain → flip → verify → rollback rollout with the probe-host false-pass caveat. Fleet runbook in [SEAL-CHECK.md](SEAL-CHECK.md). --- diff --git a/scripts/tests/e2e-auto-upgrade.sh b/scripts/tests/e2e-auto-upgrade.sh index 5ba51a60..23b5807b 100755 --- a/scripts/tests/e2e-auto-upgrade.sh +++ b/scripts/tests/e2e-auto-upgrade.sh @@ -155,16 +155,38 @@ BASELINE_PROD_DIGEST="$(helm get values "$NS" -n "$NS" --all -o json \ | jq -r '.images.ingestor.prodDigest // ""')" echo " baseline prod ingestor pin: ${BASELINE_PROD_DIGEST:-}" +# The baseline (published) release's RENDERED egress posture, captured the same +# way and for the same reason as the prod pin above: path 1's --reuse-values +# replays the baseline's computed values verbatim, so what it must assert is +# "unchanged from the baseline", NOT a hardcoded era. While the published +# baseline is still permissive this is the external-443 rule present + no +# EGRESS_PROXY_URL; once THIS chart (RFC-0003 D6 deny-by-default defaults) is +# the published baseline, --reuse-values replays lockdown instead — 443 gone, +# gateway routed. Read the posture from the installed baseline so the assertion +# is correct in both eras rather than tripping the moment the default flips +# (Bugbot on this PR). +if netpol_has_external_443; then BASELINE_EXTERNAL_443=1; else BASELINE_EXTERNAL_443=0; fi +BASELINE_EGRESS_PROXY_URL="$(jm_egress_proxy_url)" +echo " baseline egress posture: external_443=$([ "$BASELINE_EXTERNAL_443" = 1 ] && echo present || echo absent) egress_proxy_url=${BASELINE_EGRESS_PROXY_URL:-}" + echo "── simulate an image-refresh-managed annotation (must survive upgrades) ──" kubectl annotate -n "$NS" "$(jm_deploy)" \ "tracebloc.io/last-refreshed-jobs-manager-digest=sha256:e2e-sentinel" --overwrite echo "── path 1: manual-operator habit — helm upgrade --reuse-values ──" -# Old stored values replayed against the new chart: every new key is absent. -# The nil-guards must hold, and the lockdown must NOT engage by accident. +# Old stored values replayed against the new chart: every new key is absent and +# the nil-guards must hold. The egress posture must be REPLAYED FROM THE BASELINE +# VERBATIM — not moved by the new chart's defaults in either direction. That is +# baseline-derived (like the prod pin below), so the assertion holds whether the +# baseline is permissive (443 present, no gateway) or already deny-by-default +# (443 gone, gateway routed) once this chart is published. helm upgrade "$NS" "$CHART_DIR" --namespace "$NS" --reuse-values -netpol_has_external_443 || fail "--reuse-values upgrade dropped the external 443 rule (lockdown engaged by accident)" -[ -z "$(jm_egress_proxy_url)" ] || fail "--reuse-values upgrade injected EGRESS_PROXY_URL (routing engaged by accident)" +if [ "$BASELINE_EXTERNAL_443" = 1 ]; then + netpol_has_external_443 || fail "--reuse-values dropped the baseline's external 443 rule (new deny-by-default default leaked in; --reuse-values must replay the baseline verbatim)" +else + netpol_has_external_443 && fail "--reuse-values added an external 443 rule the deny-by-default baseline did not have (--reuse-values must replay the baseline verbatim)" +fi +[ "$(jm_egress_proxy_url)" = "$BASELINE_EGRESS_PROXY_URL" ] || fail "--reuse-values did not replay the baseline EGRESS_PROXY_URL verbatim: got '$(jm_egress_proxy_url)', want '${BASELINE_EGRESS_PROXY_URL:-}' (new routeWorkloads default leaked in on this path)" # Documented limitation, asserted so it stays a known quantity: plain # --reuse-values replays the old release's COMPUTED values and ignores the new # chart's defaults. What that means for the prod ingestor pin depends on the @@ -181,7 +203,7 @@ else [ "$(jm_ingestor_digest)" = "$BASELINE_PROD_DIGEST" ] \ || fail "--reuse-values did not replay the baseline prod pin verbatim: got '$(jm_ingestor_digest)', want '$BASELINE_PROD_DIGEST' (stored computed values must win over new chart defaults on this path)" fi -echo " OK: upgrade succeeded, lockdown stayed off, ingestor pin matches the baseline era (${BASELINE_PROD_DIGEST:-floating})" +echo " OK: upgrade succeeded, egress posture replayed from the baseline verbatim, ingestor pin matches the baseline era (${BASELINE_PROD_DIGEST:-floating})" echo "── isolate path 2 from path 1's --reuse-values contamination (#459) ──" # path 1's --reuse-values rewrote THIS release's recorded values to the baseline's FULL @@ -215,8 +237,8 @@ echo " OK: recorded values reset to the genuine overrides — path 2 now tests echo "── path 2: the fleet auto-upgrade — helm upgrade --reset-then-reuse-values ──" helm upgrade "$NS" "$CHART_DIR" --namespace "$NS" --reset-then-reuse-values -netpol_has_external_443 || fail "auto-upgrade dropped the external 443 rule (allowExternalHttps default did not flow)" -[ -z "$(jm_egress_proxy_url)" ] || fail "auto-upgrade injected EGRESS_PROXY_URL (routeWorkloads should default false)" +netpol_has_external_443 && fail "auto-upgrade kept the external 443 rule (deny-by-default allowExternalHttps=false did not flow — RFC-0003 D6 / client-runtime#199)" +[ "$(jm_egress_proxy_url)" = "http://egress-proxy-service:3128" ] || fail "auto-upgrade did not inject EGRESS_PROXY_URL (routeWorkloads=true default did not flow — RFC-0003 D6 / client-runtime#199)" kubectl get deploy "${NS}-egress-proxy" -n "$NS" >/dev/null \ || fail "auto-upgrade did not deploy the egress gateway (new defaults did not flow)" ANNOT="$(kubectl get -n "$NS" "$(jm_deploy)" \ @@ -245,32 +267,43 @@ WANT_DIGEST="$(local_prod_digest)" GOT_DIGEST="$(jm_ingestor_digest)" [ "$GOT_DIGEST" = "$WANT_DIGEST" ] \ || fail "auto-upgrade did not push the prod ingestor pin onto the installed edge: got '${GOT_DIGEST:-}', want '$WANT_DIGEST' (backend#1245)" -echo " OK: new defaults flowed in (gateway deployed, inert), annotations survived" +echo " OK: new defaults flowed in (deny-by-default: gateway routing + external-443 dropped), annotations survived" echo " OK: prod ingestor pin reached the installed edge ($WANT_DIGEST)" -echo "── path 3: operator flips the #102 lockdown + opts a canary off the prod pin ──" +echo "── path 3: operator OPTS OUT of the deny-by-default lockdown + opts a canary off the prod pin ──" +# Every --set value is the OPPOSITE of the chart default on purpose (Bugbot on +# this PR): now that deny-by-default ships, re-setting the lockdown values would +# just match the defaults, so path 4 could not tell a preserved override from a +# plain default replay. So opt a fleet fully out — routeWorkloads=false (stop +# routing egress through the gateway) AND allowExternalHttps=true (re-open the +# direct external-443 rule) — both genuine overrides, so path 4 becomes a real +# test that --reset-then-reuse-values keeps each opt-out instead of the next +# hourly auto-upgrade silently re-locking a fleet that deliberately opted out. helm upgrade "$NS" "$CHART_DIR" --namespace "$NS" --reset-then-reuse-values \ - --set egressProxy.routeWorkloads=true \ - --set networkPolicy.training.allowExternalHttps=false \ + --set egressProxy.routeWorkloads=false \ + --set networkPolicy.training.allowExternalHttps=true \ --set images.ingestor.prodPin=false -netpol_has_external_443 && fail "lockdown flip did NOT drop the external 443 rule" -[ "$(jm_egress_proxy_url)" = "http://egress-proxy-service:3128" ] \ - || fail "lockdown flip did not point jobs-manager at the egress gateway" +netpol_has_external_443 \ + || fail "operator opt-out did NOT re-open the external 443 rule (allowExternalHttps=true was ignored)" +[ -z "$(jm_egress_proxy_url)" ] \ + || fail "operator opt-out did NOT stop gateway routing (routeWorkloads=false was ignored — EGRESS_PROXY_URL still injected)" [ -z "$(jm_ingestor_digest)" ] \ || fail "prodPin=false did not float the canary edge back onto the ingestor tag (backend#1245)" -echo " OK: rule 2 dropped, training pods route via the gateway, canary floats" +echo " OK: rule 2 re-opened + gateway routing off by the opt-out, canary floats" -echo "── path 4: the NEXT hourly auto-upgrade must preserve both overrides ──" +echo "── path 4: the NEXT hourly auto-upgrade must preserve both opt-outs ──" +# Both overrides are user-supplied and OPPOSITE the chart default, so +# --reset-then-reuse-values must replay them: an operator who opted a fleet out +# of the lockdown (or floated a canary) must not be silently reverted to the +# deny-by-default / re-pinned by the next hourly upgrade. helm upgrade "$NS" "$CHART_DIR" --namespace "$NS" --reset-then-reuse-values -netpol_has_external_443 && fail "auto-upgrade after the flip re-opened the external 443 rule (override lost)" -[ "$(jm_egress_proxy_url)" = "http://egress-proxy-service:3128" ] \ - || fail "auto-upgrade after the flip lost EGRESS_PROXY_URL (override lost)" -# The canary opt-out is user-supplied, so --reset-then-reuse-values must replay -# it: an edge deliberately floated must not be silently re-pinned by the next -# hourly upgrade. +netpol_has_external_443 \ + || fail "auto-upgrade reverted the operator's allowExternalHttps=true opt-out back to the deny-by-default (override lost)" +[ -z "$(jm_egress_proxy_url)" ] \ + || fail "auto-upgrade re-injected EGRESS_PROXY_URL, reverting the operator's routeWorkloads=false opt-out (override lost)" [ -z "$(jm_ingestor_digest)" ] \ || fail "auto-upgrade re-pinned an edge the operator had opted out with prodPin=false (override lost)" -echo " OK: the operator's lockdown and canary opt-out persist across auto-upgrades" +echo " OK: the operator's egress opt-outs (routing + external-443) and canary opt-out persist across auto-upgrades" echo "── path 5: client credentials resolve from the existing Secret (backend#2571) ──" # THE ONLY PLACE THIS MECHANISM CAN BE TESTED. secrets.yaml resolves diff --git a/scripts/tests/pod-qos-expect.csi.txt b/scripts/tests/pod-qos-expect.csi.txt index c8e338b6..a21648d6 100644 --- a/scripts/tests/pod-qos-expect.csi.txt +++ b/scripts/tests/pod-qos-expect.csi.txt @@ -5,6 +5,7 @@ # comments on exactly the clusters the installer provisions (backend#2872). class mysql-client Burstable class t-auto-upgrade Guaranteed +class t-egress-enforcement-check Burstable class t-egress-proxy Burstable class t-egress-reachability-check Burstable class t-image-refresh Guaranteed diff --git a/scripts/tests/pod-qos-expect.gpu-amd.txt b/scripts/tests/pod-qos-expect.gpu-amd.txt index 8c245e65..453d8346 100644 --- a/scripts/tests/pod-qos-expect.gpu-amd.txt +++ b/scripts/tests/pod-qos-expect.gpu-amd.txt @@ -13,6 +13,7 @@ class amdgpu-device-plugin-daemonset Guaranteed class mysql-client Burstable class t-auto-upgrade Guaranteed +class t-egress-enforcement-check Burstable class t-egress-proxy Burstable class t-egress-reachability-check Burstable class t-image-refresh Guaranteed diff --git a/scripts/tests/pod-qos-expect.gpu-nvidia.txt b/scripts/tests/pod-qos-expect.gpu-nvidia.txt index 4ec08091..e94f3c68 100644 --- a/scripts/tests/pod-qos-expect.gpu-nvidia.txt +++ b/scripts/tests/pod-qos-expect.gpu-nvidia.txt @@ -13,6 +13,7 @@ class nvidia-device-plugin-daemonset Guaranteed class mysql-client Burstable class t-auto-upgrade Guaranteed +class t-egress-enforcement-check Burstable class t-egress-proxy Burstable class t-egress-reachability-check Burstable class t-image-refresh Guaranteed diff --git a/scripts/tests/pod-qos-expect.hostpath.txt b/scripts/tests/pod-qos-expect.hostpath.txt index 14fecfb0..5e3fc568 100644 --- a/scripts/tests/pod-qos-expect.hostpath.txt +++ b/scripts/tests/pod-qos-expect.hostpath.txt @@ -11,6 +11,7 @@ # for the two people keep trying to fix. class mysql-client Burstable class t-auto-upgrade Guaranteed +class t-egress-enforcement-check Burstable class t-egress-proxy Burstable class t-egress-reachability-check Burstable class t-image-refresh Guaranteed diff --git a/scripts/tests/qos-reachability-expect.txt b/scripts/tests/qos-reachability-expect.txt index ad1c74b3..f6215571 100644 --- a/scripts/tests/qos-reachability-expect.txt +++ b/scripts/tests/qos-reachability-expect.txt @@ -58,6 +58,8 @@ mysql-client hostPath=false blocked mysql-format-guard mysql-client hostPath=true blocked init-mysql-data,mysql-format-guard t-auto-upgrade hostPath=false reachable - t-auto-upgrade hostPath=true reachable - +t-egress-enforcement-check hostPath=false blocked probe +t-egress-enforcement-check hostPath=true blocked probe t-egress-proxy hostPath=false reachable - t-egress-proxy hostPath=true reachable - t-egress-reachability-check hostPath=false blocked probe