diff --git a/client/Chart.yaml b/client/Chart.yaml index d79f4213..d28d7c83 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.91 -appVersion: "1.9.91" +version: 1.9.92 +appVersion: "1.9.92" keywords: - tracebloc - kubernetes diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 8330a36f..5f15bdd4 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -48,6 +48,19 @@ spec: # It never reaches those spawned writers, so it's all risk and no gain. seccompProfile: type: RuntimeDefault + # backend#2913: gate the api container on MySQL accepting connections. Its + # jobs_manager.py makes an INITIAL, un-retried connect at startup; a MySQL + # that is not listening yet (errno 111, connection refused) makes that + # connect raise and the process exit, so the kubelet crashloops the pod and + # an e2e leg fails before it reaches training — and because the pod recovers + # minutes later, the only evidence is the PREVIOUS container's log. The chart + # already sequences its other MySQL clients (requests-proxy retries its + # bootstrap in-image); the api container had no such gate. This is a + # chart-level wait because the retry does not live in the jobs-manager image, + # and putting it here also protects every non-e2e install. wait-for-mysql is + # appended AFTER init-writable-data below, so the hostPath-only perm fix keeps + # index 0 and only ever runs against the local dirs. + initContainers: {{- if (default dict .Values.hostPath).enabled }} # kubelet does NOT apply fsGroup to hostPath volumes (kubernetes/kubernetes#138411), # so /data/shared AND /data/logs are created root-owned and non-root pods can't write @@ -106,7 +119,6 @@ spec: # divergence: the chart does NOT send chown/chmod stderr to /dev/null, so the real errno # (Operation not permitted vs Read-only file system) lands in `kubectl logs` beside the # verdict; the installer suppresses it because its output is a user-facing progress line. - initContainers: - name: init-writable-data image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} securityContext: @@ -152,6 +164,81 @@ spec: - name: logs-volume mountPath: /data/logs {{- end }} + # backend#2913: always-on (both hostPath and CSI installs) — the MySQL + # dependency exists on every install, unlike the perm fix above. Bounded + # retry: it loops until mysql-client:3306 accepts a TCP connection, then + # exits 0; past the cap it exits non-zero so a genuinely-down MySQL surfaces + # as a LOUD init failure rather than an infinite Pending or a silent + # crashloop. The cap (80 attempts 3s apart: ~240s of sleeping when the port + # is refused, up to ~480s if a connect hangs) sits above MySQL's own + # worst-case startup — its startupProbe tolerates ~130s on a fresh datadir + # (mysql-deployment.yaml: initialDelaySeconds 10 + failureThreshold 24 * + # periodSeconds 5). A TCP connect through the mysql-client Service is the + # right signal, and for a STRUCTURAL reason, not just because errno 111 means + # "not listening": mysql-client is a plain selector Service with no + # publishNotReadyAddresses, so its EndpointSlice carries ONLY Ready addresses + # (mysql-service.yaml). A successful connect therefore proves mysqld passed + # its mysqladmin-ping readinessProbe (mysql-deployment.yaml), not merely that + # a socket is open — so adding publishNotReadyAddresses to that Service later + # would quietly downgrade this gate to a bare TCP probe. Same + # busybox image as init-writable-data, but non-root — it needs only the + # network, no writes; busybox has no baked non-root user, so runAsUser is set + # explicitly to satisfy the pod's runAsNonRoot. + - name: wait-for-mysql + image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + securityContext: + runAsUser: 1000 + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + # requests == limits on BOTH dimensions, deliberately (Saqlain/Asad on + # client#941). On CSI this is the pod's FIRST init container + # (init-writable-data is hostPath-only), so an UNRESOURCED init here would + # silently forbid Guaranteed QoS on a profile whose api/pods-monitor + # requests==limits are equalised (e.g. eks-values.yaml): the pod would + # render Burstable instead, with no error and no log line — the exact + # regression client#922's pod-qos goldens exist to catch. Equalising this + # init keeps Guaranteed reachable, leaves the default Burstable (api's cpu + # is 250m/1000m regardless), and costs effectively nothing: Kubernetes + # takes the MAX of init requests against the SUM of the app containers, and + # 10m sits far under api's 250m, so the pod's effective request does not + # move. It also keeps the pod admissible under a ResourceQuota on + # requests.cpu, which would otherwise reject it outright on CSI. + resources: + requests: + cpu: "10m" + memory: "16Mi" + limits: + cpu: "10m" + memory: "16Mi" + command: + - 'sh' + - '-c' + - | + host=mysql-client port=3306 + # 80 attempts, 3s apart: ~240s of sleeping in the refused case (the + # errno-111 bug), up to ~480s if a connect hangs on -w 3. Both sit + # above MySQL's ~130s worst-case startup; see the YAML comment above. + attempts=80 interval=3 + i=1 + while [ "$i" -le "$attempts" ]; do + # busybox nc: -z scan (no I/O), -w 3 per-connect timeout. exit 0 = + # listening. Before mysql-client's Service/endpoint exists the name + # may not resolve — that is just another not-ready attempt. + if nc -z -w 3 "$host" "$port" 2>/dev/null; then + echo "wait-for-mysql: $host:$port is accepting connections (attempt $i/$attempts)" + exit 0 + fi + echo "wait-for-mysql: $host:$port not ready yet (attempt $i/$attempts), retrying in ${interval}s" + sleep "$interval" + i=$((i+1)) + done + echo "wait-for-mysql: FATAL $host:$port did not accept a connection after $attempts attempts -- refusing to start jobs-manager against a MySQL that is not listening. Check the mysql-client pod: kubectl get pods -l app=mysql-client" >&2 + exit 1 containers: - name: api image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 0d6c78da..2e3ddf1a 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -830,6 +830,9 @@ tests: path: spec.template.spec.initContainers[0].command[2] pattern: '(?m)^exit 0$' + # On CSI the privileged init-writable-data is still skipped, but wait-for-mysql + # (backend#2913) is always-on, so it is now the ONLY initContainer here — not + # the empty list this used to assert. - it: "CSI install (hostPath disabled) skips the privileged init and sets no fsGroup (#611)" set: hostPath: @@ -837,8 +840,89 @@ tests: asserts: - notExists: path: spec.template.spec.securityContext.fsGroup + # exactly one initContainer, and it is the mysql gate — not the privileged perm fix + - lengthEqual: + path: spec.template.spec.initContainers + count: 1 + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-mysql - notExists: + path: spec.template.spec.initContainers[1] + + # backend#2913: the api container's jobs_manager.py makes an INITIAL, un-retried + # MySQL connect at startup; a MySQL not yet listening (errno 111) makes it raise + # and the pod crashloops, failing an e2e leg before training. This chart-level + # initContainer waits for mysql-client:3306 before the api container starts. + - it: "waits for MySQL before starting the api container (backend#2913)" + asserts: + # always-on: present even on a default (CSI, hostPath disabled) install + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-mysql + # non-root, no caps, read-only rootfs — it needs only the network + - equal: + path: spec.template.spec.initContainers[0].securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.initContainers[0].securityContext.runAsUser + value: 1000 + - equal: + path: spec.template.spec.initContainers[0].securityContext.readOnlyRootFilesystem + value: true + - equal: + path: spec.template.spec.initContainers[0].securityContext.capabilities.drop + value: ["ALL"] + # requests == limits on BOTH dimensions (client#941): an unresourced init + # would forbid Guaranteed QoS on CSI (where this is the only init container) + # and collide with client#922's pod-qos goldens. Guard the equality directly. + - equal: + path: spec.template.spec.initContainers[0].resources.requests.cpu + value: "10m" + - equal: + path: spec.template.spec.initContainers[0].resources.limits.cpu + value: "10m" + - equal: + path: spec.template.spec.initContainers[0].resources.requests.memory + value: "16Mi" + - equal: + path: spec.template.spec.initContainers[0].resources.limits.memory + value: "16Mi" + # gates on the mysql Service host:port the api container also uses + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'host=mysql-client port=3306' + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'nc -z -w 3 "\$host" "\$port"' + # BOUNDED, then fails LOUDLY: a genuinely-down MySQL must surface as an init + # error, not an infinite Pending or a silent crashloop (issue "Do" item 1). + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'attempts=80 interval=3' + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'while \[ "\$i" -le "\$attempts" \]' + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: '(?s)wait-for-mysql: FATAL.*exit 1' + + # hostPath install: the perm fix keeps index 0 (its tests pin [0]), and + # wait-for-mysql is appended after it — so BOTH init containers run, in order. + - it: "hostPath install runs both init-writable-data and wait-for-mysql, in order (backend#2913)" + set: + hostPath: + enabled: true + asserts: + - lengthEqual: path: spec.template.spec.initContainers + count: 2 + - equal: + path: spec.template.spec.initContainers[0].name + value: init-writable-data + - equal: + path: spec.template.spec.initContainers[1].name + value: wait-for-mysql # backend#1528 S1: the tb_meta / tb_ingest env is flag-gated on # serviceDbAccounts so a default install is byte-identical. diff --git a/scripts/tests/jobs-manager-waits-for-mysql.bats b/scripts/tests/jobs-manager-waits-for-mysql.bats new file mode 100644 index 00000000..721f0a76 --- /dev/null +++ b/scripts/tests/jobs-manager-waits-for-mysql.bats @@ -0,0 +1,129 @@ +#!/usr/bin/env bats +# jobs-manager-waits-for-mysql.bats — behavioural guard for the wait-for-mysql +# initContainer in client/templates/jobs-manager-deployment.yaml (backend#2913). +# +# WHY THIS EXISTS +# The api container's jobs_manager.py makes an INITIAL, un-retried MySQL connect +# at startup. On an install where mysqld is not listening yet (errno 111, +# connection refused — NOT a credential problem) that connect raises and the +# process exits, so the kubelet crashloops the pod and an e2e leg fails before it +# ever reaches training. The pod recovers minutes later and looks healthy by the +# time anyone inspects, so the only evidence is the PREVIOUS container's log — +# which is what made this expensive to diagnose. The fix is a chart-level init +# gate (the retry does not live in the jobs-manager image), and the behaviour +# that matters is the not-ready-THEN-ready handoff plus the bounded, loud failure +# when MySQL never comes up. +# +# TESTS THE REAL SCRIPT, NOT A COPY (rule 1: derive, don't restate). The wait +# loop is extracted from the RENDERED chart and executed against a fake `nc` +# (and a no-op `sleep`) on PATH, exactly as wait-for-ingest-pod.bats drives a +# fake `kubectl`. A hand-copied loop here would pass while the chart's own drifted. + +setup() { + CHART="${BATS_TEST_DIRNAME}/../../client" + VALUES="${CHART}/ci/bm-values.yaml" + require_tool helm || return 1 + require_tool python3 || return 1 + require_pymodule yaml PyYAML || return 1 + [ -d "$CHART" ] || { echo "chart directory not found at $CHART" >&2; return 1; } + [ -f "$VALUES" ] || { echo "CI values file not found at $VALUES" >&2; return 1; } + + TMP="$(mktemp -d)" + BIN="$TMP/bin"; mkdir -p "$BIN" + NCCALLS="$TMP/nc_calls"; printf '0' >"$NCCALLS" + # READY_AT: the attempt on which the fake `nc` starts succeeding. A value the + # loop can never reach (999) is "MySQL never comes up". + READY_AT="$TMP/ready_at"; printf '999' >"$READY_AT" + + # Fake `nc`: ignores every flag/host/port the script passes and answers purely + # from the counter, so a test controls exactly when the port "opens". exit 0 = + # listening, non-zero = connection refused (errno 111), which is the case. + cat >"$BIN/nc" <"$NCCALLS" +[ "\$n" -ge "\$(cat "$READY_AT")" ] && exit 0 || exit 1 +EOF + # No-op `sleep` so the bounded-failure case runs its whole cap instantly + # instead of taking the real 240s. + printf '#!/usr/bin/env bash\nexit 0\n' >"$BIN/sleep" + chmod +x "$BIN/nc" "$BIN/sleep" + + # Extract with the REAL tools on PATH (helm, python3)... + SCRIPT="$TMP/wait.sh" + extract_wait_for_mysql >"$SCRIPT" + [ -s "$SCRIPT" ] || { echo "could not extract the wait-for-mysql command from the rendered chart" >&2; return 1; } + + # ...then put the fakes ahead of the real tools before the wait loop runs. + # Without this the loop would hit real DNS/connect on a non-existent host and + # the bounded-failure case would take its full real 240s cap. + PATH="$BIN:$PATH" +} + +teardown() { [ -n "${TMP:-}" ] && rm -rf "$TMP"; } + +# A missing tool is a SKIP on a laptop and a FAILURE in CI — a required gate must +# never report green on assertions it silently skipped (see chart-pull-secret.bats). +require_tool() { + command -v "$1" >/dev/null && return 0 + if [ "${CI:-}" = "true" ]; then + echo "::error::$1 is missing in CI, so this chart-render guard would be" >&2 + echo "::error::skipped rather than run. Install it in the job instead." >&2 + return 1 + fi + skip "$1 not installed (local run)" +} + +require_pymodule() { + python3 -c "import $1" >/dev/null 2>&1 && return 0 + if [ "${CI:-}" = "true" ]; then + echo "::error::python3 module '$1' ($2) is missing in CI (pip install $2)." >&2 + return 1 + fi + skip "python3 module '$1' ($2) not installed (local run)" +} + +# Render the REAL chart and pull out the wait-for-mysql initContainer's shell +# body (command[2]). Parsed as YAML, not grepped: there are two init containers +# and two app containers, and only a structured read can pick the right one by +# name regardless of its index (it is [0] on CSI, [1] on hostPath). +extract_wait_for_mysql() { + helm template myrel "$CHART" -f "$VALUES" --namespace tracebloc \ + -s templates/jobs-manager-deployment.yaml 2>/dev/null | python3 -c ' +import sys, yaml +for doc in yaml.safe_load_all(sys.stdin.read()): + if not doc or doc.get("kind") != "Deployment": + continue + if "jobs-manager" not in doc["metadata"]["name"]: + continue + for c in (doc["spec"]["template"]["spec"].get("initContainers") or []): + if c["name"] == "wait-for-mysql": + sys.stdout.write(c["command"][2]) +' +} + +@test "hands off to the api container once MySQL starts accepting connections" { + printf '3' >"$READY_AT" # refused twice, then listening on the 3rd attempt + run sh "$SCRIPT" + [ "$status" -eq 0 ] || { echo "expected exit 0, got $status: $output" >&2; return 1; } + [[ "$output" == *"not ready yet (attempt 1/80)"* ]] || return 1 + [[ "$output" == *"is accepting connections (attempt 3/80)"* ]] || return 1 +} + +@test "the very first probe succeeding hands off immediately (no spurious wait)" { + printf '1' >"$READY_AT" + run sh "$SCRIPT" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"is accepting connections (attempt 1/80)"* ]] || return 1 + [[ "$output" != *"not ready yet"* ]] || return 1 +} + +@test "is BOUNDED and fails LOUDLY when MySQL never comes up (not an infinite Pending)" { + # READY_AT stays 999 — the port never opens. The loop must give up at the cap + # with a non-zero exit and a diagnosable message, not spin forever. + run sh "$SCRIPT" + [ "$status" -eq 1 ] || { echo "expected exit 1, got $status" >&2; return 1; } + [[ "$output" == *"FATAL"* ]] || return 1 + [[ "$output" == *"is not listening"* ]] || return 1 + # It actually exhausted the cap rather than bailing early. + [[ "$output" == *"attempt 80/80"* ]] || return 1 +}