fix(chart): wait for MySQL before starting jobs-manager (backend#2913)

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.91
-appVersion: "1.9.91"
+version: 1.9.92
+appVersion: "1.9.92"
 keywords:
   - tracebloc
   - kubernetes

client/templates/jobs-manager-deployment.yaml

@@ -48,6 +48,19 @@ spec:
         # It never reaches those spawned writers, so it's all risk and no gain.
         seccompProfile:
           type: RuntimeDefault
+      # backend#2913: gate the api container on MySQL accepting connections. Its
+      # jobs_manager.py makes an INITIAL, un-retried connect at startup; a MySQL
+      # that is not listening yet (errno 111, connection refused) makes that
+      # connect raise and the process exit, so the kubelet crashloops the pod and
+      # an e2e leg fails before it reaches training — and because the pod recovers
+      # minutes later, the only evidence is the PREVIOUS container's log. The chart
+      # already sequences its other MySQL clients (requests-proxy retries its
+      # bootstrap in-image); the api container had no such gate. This is a
+      # chart-level wait because the retry does not live in the jobs-manager image,
+      # and putting it here also protects every non-e2e install. wait-for-mysql is
+      # appended AFTER init-writable-data below, so the hostPath-only perm fix keeps
+      # index 0 and only ever runs against the local dirs.
+      initContainers:
       {{- if (default dict .Values.hostPath).enabled }}
       # kubelet does NOT apply fsGroup to hostPath volumes (kubernetes/kubernetes#138411),
       # so /data/shared AND /data/logs are created root-owned and non-root pods can't write
@@ -106,7 +119,6 @@ spec:
       # divergence: the chart does NOT send chown/chmod stderr to /dev/null, so the real errno
       # (Operation not permitted vs Read-only file system) lands in `kubectl logs` beside the
       # verdict; the installer suppresses it because its output is a user-facing progress line.
-      initContainers:
       - name: init-writable-data
         image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
         securityContext:
@@ -152,6 +164,81 @@ spec:
         - name: logs-volume
           mountPath: /data/logs
       {{- end }}
+      # backend#2913: always-on (both hostPath and CSI installs) — the MySQL
+      # dependency exists on every install, unlike the perm fix above. Bounded
+      # retry: it loops until mysql-client:3306 accepts a TCP connection, then
+      # exits 0; past the cap it exits non-zero so a genuinely-down MySQL surfaces
+      # as a LOUD init failure rather than an infinite Pending or a silent
+      # crashloop. The cap (80 attempts 3s apart: ~240s of sleeping when the port
+      # is refused, up to ~480s if a connect hangs) sits above MySQL's own
+      # worst-case startup — its startupProbe tolerates ~130s on a fresh datadir
+      # (mysql-deployment.yaml: initialDelaySeconds 10 + failureThreshold 24 *
+      # periodSeconds 5). A TCP connect through the mysql-client Service is the
+      # right signal, and for a STRUCTURAL reason, not just because errno 111 means
+      # "not listening": mysql-client is a plain selector Service with no
+      # publishNotReadyAddresses, so its EndpointSlice carries ONLY Ready addresses
+      # (mysql-service.yaml). A successful connect therefore proves mysqld passed
+      # its mysqladmin-ping readinessProbe (mysql-deployment.yaml), not merely that
+      # a socket is open — so adding publishNotReadyAddresses to that Service later
+      # would quietly downgrade this gate to a bare TCP probe. Same
+      # busybox image as init-writable-data, but non-root — it needs only the
+      # network, no writes; busybox has no baked non-root user, so runAsUser is set
+      # explicitly to satisfy the pod's runAsNonRoot.
+      - name: wait-for-mysql
+        image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
+        securityContext:
+          runAsUser: 1000
+          runAsNonRoot: true
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop: ["ALL"]
+          seccompProfile:
+            type: RuntimeDefault
+        # requests == limits on BOTH dimensions, deliberately (Saqlain/Asad on
+        # client#941). On CSI this is the pod's FIRST init container
+        # (init-writable-data is hostPath-only), so an UNRESOURCED init here would
+        # silently forbid Guaranteed QoS on a profile whose api/pods-monitor
+        # requests==limits are equalised (e.g. eks-values.yaml): the pod would
+        # render Burstable instead, with no error and no log line — the exact
+        # regression client#922's pod-qos goldens exist to catch. Equalising this
+        # init keeps Guaranteed reachable, leaves the default Burstable (api's cpu
+        # is 250m/1000m regardless), and costs effectively nothing: Kubernetes
+        # takes the MAX of init requests against the SUM of the app containers, and
+        # 10m sits far under api's 250m, so the pod's effective request does not
+        # move. It also keeps the pod admissible under a ResourceQuota on
+        # requests.cpu, which would otherwise reject it outright on CSI.
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "16Mi"
+          limits:
+            cpu: "10m"
+            memory: "16Mi"
+        command:
+          - 'sh'
+          - '-c'
+          - |
+            host=mysql-client port=3306
+            # 80 attempts, 3s apart: ~240s of sleeping in the refused case (the
+            # errno-111 bug), up to ~480s if a connect hangs on -w 3. Both sit
+            # above MySQL's ~130s worst-case startup; see the YAML comment above.
+            attempts=80 interval=3
+            i=1
+            while [ "$i" -le "$attempts" ]; do
+              # busybox nc: -z scan (no I/O), -w 3 per-connect timeout. exit 0 =
+              # listening. Before mysql-client's Service/endpoint exists the name
+              # may not resolve — that is just another not-ready attempt.
+              if nc -z -w 3 "$host" "$port" 2>/dev/null; then
+                echo "wait-for-mysql: $host:$port is accepting connections (attempt $i/$attempts)"
+                exit 0
+              fi
+              echo "wait-for-mysql: $host:$port not ready yet (attempt $i/$attempts), retrying in ${interval}s"
+              sleep "$interval"
+              i=$((i+1))
+            done
+            echo "wait-for-mysql: FATAL $host:$port did not accept a connection after $attempts attempts -- refusing to start jobs-manager against a MySQL that is not listening. Check the mysql-client pod: kubectl get pods -l app=mysql-client" >&2
+            exit 1
       containers:
       - name: api
         image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}

client/tests/jobs_manager_test.yaml

@@ -830,15 +830,99 @@ tests:
           path: spec.template.spec.initContainers[0].command[2]
           pattern: '(?m)^exit 0$'
 
+  # On CSI the privileged init-writable-data is still skipped, but wait-for-mysql
+  # (backend#2913) is always-on, so it is now the ONLY initContainer here — not
+  # the empty list this used to assert.
   - it: "CSI install (hostPath disabled) skips the privileged init and sets no fsGroup (#611)"
     set:
       hostPath:
         enabled: false
     asserts:
       - notExists:
           path: spec.template.spec.securityContext.fsGroup
+      # exactly one initContainer, and it is the mysql gate — not the privileged perm fix
+      - lengthEqual:
+          path: spec.template.spec.initContainers
+          count: 1
+      - equal:
+          path: spec.template.spec.initContainers[0].name
+          value: wait-for-mysql
       - notExists:
+          path: spec.template.spec.initContainers[1]
+
+  # backend#2913: the api container's jobs_manager.py makes an INITIAL, un-retried
+  # MySQL connect at startup; a MySQL not yet listening (errno 111) makes it raise
+  # and the pod crashloops, failing an e2e leg before training. This chart-level
+  # initContainer waits for mysql-client:3306 before the api container starts.
+  - it: "waits for MySQL before starting the api container (backend#2913)"
+    asserts:
+      # always-on: present even on a default (CSI, hostPath disabled) install
+      - equal:
+          path: spec.template.spec.initContainers[0].name
+          value: wait-for-mysql
+      # non-root, no caps, read-only rootfs — it needs only the network
+      - equal:
+          path: spec.template.spec.initContainers[0].securityContext.runAsNonRoot
+          value: true
+      - equal:
+          path: spec.template.spec.initContainers[0].securityContext.runAsUser
+          value: 1000
+      - equal:
+          path: spec.template.spec.initContainers[0].securityContext.readOnlyRootFilesystem
+          value: true
+      - equal:
+          path: spec.template.spec.initContainers[0].securityContext.capabilities.drop
+          value: ["ALL"]
+      # requests == limits on BOTH dimensions (client#941): an unresourced init
+      # would forbid Guaranteed QoS on CSI (where this is the only init container)
+      # and collide with client#922's pod-qos goldens. Guard the equality directly.
+      - equal:
+          path: spec.template.spec.initContainers[0].resources.requests.cpu
+          value: "10m"
+      - equal:
+          path: spec.template.spec.initContainers[0].resources.limits.cpu
+          value: "10m"
+      - equal:
+          path: spec.template.spec.initContainers[0].resources.requests.memory
+          value: "16Mi"
+      - equal:
+          path: spec.template.spec.initContainers[0].resources.limits.memory
+          value: "16Mi"
+      # gates on the mysql Service host:port the api container also uses
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: 'host=mysql-client port=3306'
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: 'nc -z -w 3 "\$host" "\$port"'
+      # BOUNDED, then fails LOUDLY: a genuinely-down MySQL must surface as an init
+      # error, not an infinite Pending or a silent crashloop (issue "Do" item 1).
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: 'attempts=80 interval=3'
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: 'while \[ "\$i" -le "\$attempts" \]'
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: '(?s)wait-for-mysql: FATAL.*exit 1'
+
+  # hostPath install: the perm fix keeps index 0 (its tests pin [0]), and
+  # wait-for-mysql is appended after it — so BOTH init containers run, in order.
+  - it: "hostPath install runs both init-writable-data and wait-for-mysql, in order (backend#2913)"
+    set:
+      hostPath:
+        enabled: true
+    asserts:
+      - lengthEqual:
           path: spec.template.spec.initContainers
+          count: 2
+      - equal:
+          path: spec.template.spec.initContainers[0].name
+          value: init-writable-data
+      - equal:
+          path: spec.template.spec.initContainers[1].name
+          value: wait-for-mysql
 
   # backend#1528 S1: the tb_meta / tb_ingest env is flag-gated on
   # serviceDbAccounts so a default install is byte-identical.

scripts/tests/jobs-manager-waits-for-mysql.bats

@@ -0,0 +1,129 @@
+#!/usr/bin/env bats
+# jobs-manager-waits-for-mysql.bats — behavioural guard for the wait-for-mysql
+# initContainer in client/templates/jobs-manager-deployment.yaml (backend#2913).
+#
+# WHY THIS EXISTS
+# The api container's jobs_manager.py makes an INITIAL, un-retried MySQL connect
+# at startup. On an install where mysqld is not listening yet (errno 111,
+# connection refused — NOT a credential problem) that connect raises and the
+# process exits, so the kubelet crashloops the pod and an e2e leg fails before it
+# ever reaches training. The pod recovers minutes later and looks healthy by the
+# time anyone inspects, so the only evidence is the PREVIOUS container's log —
+# which is what made this expensive to diagnose. The fix is a chart-level init
+# gate (the retry does not live in the jobs-manager image), and the behaviour
+# that matters is the not-ready-THEN-ready handoff plus the bounded, loud failure
+# when MySQL never comes up.
+#
+# TESTS THE REAL SCRIPT, NOT A COPY (rule 1: derive, don't restate). The wait
+# loop is extracted from the RENDERED chart and executed against a fake `nc`
+# (and a no-op `sleep`) on PATH, exactly as wait-for-ingest-pod.bats drives a
+# fake `kubectl`. A hand-copied loop here would pass while the chart's own drifted.
+
+setup() {
+  CHART="${BATS_TEST_DIRNAME}/../../client"
+  VALUES="${CHART}/ci/bm-values.yaml"
+  require_tool helm || return 1
+  require_tool python3 || return 1
+  require_pymodule yaml PyYAML || return 1
+  [ -d "$CHART" ] || { echo "chart directory not found at $CHART" >&2; return 1; }
+  [ -f "$VALUES" ] || { echo "CI values file not found at $VALUES" >&2; return 1; }
+
+  TMP="$(mktemp -d)"
+  BIN="$TMP/bin"; mkdir -p "$BIN"
+  NCCALLS="$TMP/nc_calls"; printf '0' >"$NCCALLS"
+  # READY_AT: the attempt on which the fake `nc` starts succeeding. A value the
+  # loop can never reach (999) is "MySQL never comes up".
+  READY_AT="$TMP/ready_at"; printf '999' >"$READY_AT"
+
+  # Fake `nc`: ignores every flag/host/port the script passes and answers purely
+  # from the counter, so a test controls exactly when the port "opens". exit 0 =
+  # listening, non-zero = connection refused (errno 111), which is the case.
+  cat >"$BIN/nc" <<EOF
+#!/usr/bin/env bash
+n=\$(cat "$NCCALLS"); n=\$((n + 1)); printf '%s' "\$n" >"$NCCALLS"
+[ "\$n" -ge "\$(cat "$READY_AT")" ] && exit 0 || exit 1
+EOF
+  # No-op `sleep` so the bounded-failure case runs its whole cap instantly
+  # instead of taking the real 240s.
+  printf '#!/usr/bin/env bash\nexit 0\n' >"$BIN/sleep"
+  chmod +x "$BIN/nc" "$BIN/sleep"
+
+  # Extract with the REAL tools on PATH (helm, python3)...
+  SCRIPT="$TMP/wait.sh"
+  extract_wait_for_mysql >"$SCRIPT"
+  [ -s "$SCRIPT" ] || { echo "could not extract the wait-for-mysql command from the rendered chart" >&2; return 1; }
+
+  # ...then put the fakes ahead of the real tools before the wait loop runs.
+  # Without this the loop would hit real DNS/connect on a non-existent host and
+  # the bounded-failure case would take its full real 240s cap.
+  PATH="$BIN:$PATH"
+}
+
+teardown() { [ -n "${TMP:-}" ] && rm -rf "$TMP"; }
+
+# A missing tool is a SKIP on a laptop and a FAILURE in CI — a required gate must
+# never report green on assertions it silently skipped (see chart-pull-secret.bats).
+require_tool() {
+  command -v "$1" >/dev/null && return 0
+  if [ "${CI:-}" = "true" ]; then
+    echo "::error::$1 is missing in CI, so this chart-render guard would be" >&2
+    echo "::error::skipped rather than run. Install it in the job instead." >&2
+    return 1
+  fi
+  skip "$1 not installed (local run)"
+}
+
+require_pymodule() {
+  python3 -c "import $1" >/dev/null 2>&1 && return 0
+  if [ "${CI:-}" = "true" ]; then
+    echo "::error::python3 module '$1' ($2) is missing in CI (pip install $2)." >&2
+    return 1
+  fi
+  skip "python3 module '$1' ($2) not installed (local run)"
+}
+
+# Render the REAL chart and pull out the wait-for-mysql initContainer's shell
+# body (command[2]). Parsed as YAML, not grepped: there are two init containers
+# and two app containers, and only a structured read can pick the right one by
+# name regardless of its index (it is [0] on CSI, [1] on hostPath).
+extract_wait_for_mysql() {
+  helm template myrel "$CHART" -f "$VALUES" --namespace tracebloc \
+    -s templates/jobs-manager-deployment.yaml 2>/dev/null | python3 -c '
+import sys, yaml
+for doc in yaml.safe_load_all(sys.stdin.read()):
+    if not doc or doc.get("kind") != "Deployment":
+        continue
+    if "jobs-manager" not in doc["metadata"]["name"]:
+        continue
+    for c in (doc["spec"]["template"]["spec"].get("initContainers") or []):
+        if c["name"] == "wait-for-mysql":
+            sys.stdout.write(c["command"][2])
+'
+}
+
+@test "hands off to the api container once MySQL starts accepting connections" {
+  printf '3' >"$READY_AT"   # refused twice, then listening on the 3rd attempt
+  run sh "$SCRIPT"
+  [ "$status" -eq 0 ] || { echo "expected exit 0, got $status: $output" >&2; return 1; }
+  [[ "$output" == *"not ready yet (attempt 1/80)"* ]] || return 1
+  [[ "$output" == *"is accepting connections (attempt 3/80)"* ]] || return 1
+}
+
+@test "the very first probe succeeding hands off immediately (no spurious wait)" {
+  printf '1' >"$READY_AT"
+  run sh "$SCRIPT"
+  [ "$status" -eq 0 ] || return 1
+  [[ "$output" == *"is accepting connections (attempt 1/80)"* ]] || return 1
+  [[ "$output" != *"not ready yet"* ]] || return 1
+}
+
+@test "is BOUNDED and fails LOUDLY when MySQL never comes up (not an infinite Pending)" {
+  # READY_AT stays 999 — the port never opens. The loop must give up at the cap
+  # with a non-zero exit and a diagnosable message, not spin forever.
+  run sh "$SCRIPT"
+  [ "$status" -eq 1 ] || { echo "expected exit 1, got $status" >&2; return 1; }
+  [[ "$output" == *"FATAL"* ]] || return 1
+  [[ "$output" == *"is not listening"* ]] || return 1
+  # It actually exhausted the cap rather than bailing early.
+  [[ "$output" == *"attempt 80/80"* ]] || return 1
+}