From c4ba47b6f3c8b4ac0f9d2ef0f229355076bd30d7 Mon Sep 17 00:00:00 2001 From: Arturo Peroni Date: Mon, 31 Aug 2026 17:46:59 +0200 Subject: [PATCH 1/4] fix(chart): wait for MySQL before starting jobs-manager (backend#2913) jobs-manager's api container (jobs_manager.py) makes an INITIAL, un-retried MySQL connect at startup. On an install where mysqld is not listening yet (errno 111, connection refused -- not a credential problem) that connect raises and the process exits, so the kubelet crashloops the pod and an e2e leg fails before it ever reaches training. The pod recovers minutes later and looks healthy by the time anyone inspects, so the only evidence is the PREVIOUS container's log -- which is what made this expensive to diagnose (measured on staging run 33393213667). The chart already sequences its other MySQL clients (requests-proxy retries its bootstrap in-image); the api container had no such gate. Add an always-on `wait-for-mysql` initContainer that bounded-retries a TCP connect to mysql-client:3306 (busybox `nc -z`, 80 attempts 3s apart) and then hands off, or exits non-zero past the cap so a genuinely-down MySQL surfaces as a LOUD init failure rather than an infinite Pending or a silent crashloop. It is a chart-level wait because the retry does not live in the jobs-manager image; putting it here also protects every non-e2e install. It is appended AFTER the hostPath-only init-writable-data, so that perm fix keeps index 0; on CSI (hostPath disabled) wait-for-mysql is the sole init container -- previously there were none, so busybox is now referenced on every install (already an enumerated mirror image; mirror-enumeration guard stays green). Coverage: helm-unittest asserts the gate is present, non-root, bounded and fails loudly; a new behavioural bats (jobs-manager-waits-for-mysql.bats) extracts the REAL rendered loop and runs it against a fake nc for the not-ready-then-ready handoff and the bounded loud-failure path. Out of scope (issue "Do" item 2): whether a leg that dies before observing an axis should record framework=unknown vs record nothing is an e2e-test-agent/ledger decision, split out for Lukas. Closes tracebloc/backend#2913 Co-Authored-By: Claude Opus 4.8 --- client/Chart.yaml | 4 +- client/templates/jobs-manager-deployment.yaml | 64 ++++++++- client/tests/jobs_manager_test.yaml | 69 ++++++++++ .../tests/jobs-manager-waits-for-mysql.bats | 129 ++++++++++++++++++ 4 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 scripts/tests/jobs-manager-waits-for-mysql.bats diff --git a/client/Chart.yaml b/client/Chart.yaml index 517bbae7..d79f4213 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.90 -appVersion: "1.9.90" +version: 1.9.91 +appVersion: "1.9.91" keywords: - tracebloc - kubernetes diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index e3659fea..adf85290 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -48,6 +48,19 @@ spec: # It never reaches those spawned writers, so it's all risk and no gain. seccompProfile: type: RuntimeDefault + # backend#2913: gate the api container on MySQL accepting connections. Its + # jobs_manager.py makes an INITIAL, un-retried connect at startup; a MySQL + # that is not listening yet (errno 111, connection refused) makes that + # connect raise and the process exit, so the kubelet crashloops the pod and + # an e2e leg fails before it reaches training — and because the pod recovers + # minutes later, the only evidence is the PREVIOUS container's log. The chart + # already sequences its other MySQL clients (requests-proxy retries its + # bootstrap in-image); the api container had no such gate. This is a + # chart-level wait because the retry does not live in the jobs-manager image, + # and putting it here also protects every non-e2e install. wait-for-mysql is + # appended AFTER init-writable-data below, so the hostPath-only perm fix keeps + # index 0 and only ever runs against the local dirs. + initContainers: {{- if .Values.hostPath.enabled }} # kubelet does NOT apply fsGroup to hostPath volumes (kubernetes/kubernetes#138411), # so /data/shared AND /data/logs are created root-owned and non-root pods can't write @@ -106,7 +119,6 @@ spec: # divergence: the chart does NOT send chown/chmod stderr to /dev/null, so the real errno # (Operation not permitted vs Read-only file system) lands in `kubectl logs` beside the # verdict; the installer suppresses it because its output is a user-facing progress line. - initContainers: - name: init-writable-data image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} securityContext: @@ -152,6 +164,56 @@ spec: - name: logs-volume mountPath: /data/logs {{- end }} + # backend#2913: always-on (both hostPath and CSI installs) — the MySQL + # dependency exists on every install, unlike the perm fix above. Bounded + # retry: it loops until mysql-client:3306 accepts a TCP connection, then + # exits 0; past the cap it exits non-zero so a genuinely-down MySQL surfaces + # as a LOUD init failure rather than an infinite Pending or a silent + # crashloop. The cap (80 attempts 3s apart: ~240s of sleeping when the port + # is refused, up to ~480s if a connect hangs) sits above MySQL's own + # worst-case startup — its startupProbe tolerates ~130s on a fresh datadir + # (mysql-deployment.yaml: initialDelaySeconds 10 + failureThreshold 24 * + # periodSeconds 5). A TCP connect is the right signal: errno 111 is exactly + # "not listening yet", and + # mysqld binds :3306 only once it is ready to accept connections. Same + # busybox image as init-writable-data, but non-root — it needs only the + # network, no writes; busybox has no baked non-root user, so runAsUser is set + # explicitly to satisfy the pod's runAsNonRoot. + - name: wait-for-mysql + image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + securityContext: + runAsUser: 1000 + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + command: + - 'sh' + - '-c' + - | + host=mysql-client port=3306 + # 80 attempts, 3s apart: ~240s of sleeping in the refused case (the + # errno-111 bug), up to ~480s if a connect hangs on -w 3. Both sit + # above MySQL's ~130s worst-case startup; see the YAML comment above. + attempts=80 interval=3 + i=1 + while [ "$i" -le "$attempts" ]; do + # busybox nc: -z scan (no I/O), -w 3 per-connect timeout. exit 0 = + # listening. Before mysql-client's Service/endpoint exists the name + # may not resolve — that is just another not-ready attempt. + if nc -z -w 3 "$host" "$port" 2>/dev/null; then + echo "wait-for-mysql: $host:$port is accepting connections (attempt $i/$attempts)" + exit 0 + fi + echo "wait-for-mysql: $host:$port not ready yet (attempt $i/$attempts), retrying in ${interval}s" + sleep "$interval" + i=$((i+1)) + done + echo "wait-for-mysql: FATAL $host:$port did not accept a connection after $attempts attempts -- refusing to start jobs-manager against a MySQL that is not listening. Check the mysql-client pod: kubectl get pods -l app=mysql-client" >&2 + exit 1 containers: - name: api image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" (include "tracebloc.clientEnv" .) "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 0d6c78da..752050a0 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -830,6 +830,9 @@ tests: path: spec.template.spec.initContainers[0].command[2] pattern: '(?m)^exit 0$' + # On CSI the privileged init-writable-data is still skipped, but wait-for-mysql + # (backend#2913) is always-on, so it is now the ONLY initContainer here — not + # the empty list this used to assert. - it: "CSI install (hostPath disabled) skips the privileged init and sets no fsGroup (#611)" set: hostPath: @@ -837,8 +840,74 @@ tests: asserts: - notExists: path: spec.template.spec.securityContext.fsGroup + # exactly one initContainer, and it is the mysql gate — not the privileged perm fix + - lengthEqual: + path: spec.template.spec.initContainers + count: 1 + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-mysql - notExists: + path: spec.template.spec.initContainers[1] + + # backend#2913: the api container's jobs_manager.py makes an INITIAL, un-retried + # MySQL connect at startup; a MySQL not yet listening (errno 111) makes it raise + # and the pod crashloops, failing an e2e leg before training. This chart-level + # initContainer waits for mysql-client:3306 before the api container starts. + - it: "waits for MySQL before starting the api container (backend#2913)" + asserts: + # always-on: present even on a default (CSI, hostPath disabled) install + - equal: + path: spec.template.spec.initContainers[0].name + value: wait-for-mysql + # non-root, no caps, read-only rootfs — it needs only the network + - equal: + path: spec.template.spec.initContainers[0].securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.initContainers[0].securityContext.runAsUser + value: 1000 + - equal: + path: spec.template.spec.initContainers[0].securityContext.readOnlyRootFilesystem + value: true + - equal: + path: spec.template.spec.initContainers[0].securityContext.capabilities.drop + value: ["ALL"] + # gates on the mysql Service host:port the api container also uses + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'host=mysql-client port=3306' + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'nc -z -w 3 "\$host" "\$port"' + # BOUNDED, then fails LOUDLY: a genuinely-down MySQL must surface as an init + # error, not an infinite Pending or a silent crashloop (issue "Do" item 1). + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'attempts=80 interval=3' + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: 'while \[ "\$i" -le "\$attempts" \]' + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: '(?s)wait-for-mysql: FATAL.*exit 1' + + # hostPath install: the perm fix keeps index 0 (its tests pin [0]), and + # wait-for-mysql is appended after it — so BOTH init containers run, in order. + - it: "hostPath install runs both init-writable-data and wait-for-mysql, in order (backend#2913)" + set: + hostPath: + enabled: true + asserts: + - lengthEqual: path: spec.template.spec.initContainers + count: 2 + - equal: + path: spec.template.spec.initContainers[0].name + value: init-writable-data + - equal: + path: spec.template.spec.initContainers[1].name + value: wait-for-mysql # backend#1528 S1: the tb_meta / tb_ingest env is flag-gated on # serviceDbAccounts so a default install is byte-identical. diff --git a/scripts/tests/jobs-manager-waits-for-mysql.bats b/scripts/tests/jobs-manager-waits-for-mysql.bats new file mode 100644 index 00000000..721f0a76 --- /dev/null +++ b/scripts/tests/jobs-manager-waits-for-mysql.bats @@ -0,0 +1,129 @@ +#!/usr/bin/env bats +# jobs-manager-waits-for-mysql.bats — behavioural guard for the wait-for-mysql +# initContainer in client/templates/jobs-manager-deployment.yaml (backend#2913). +# +# WHY THIS EXISTS +# The api container's jobs_manager.py makes an INITIAL, un-retried MySQL connect +# at startup. On an install where mysqld is not listening yet (errno 111, +# connection refused — NOT a credential problem) that connect raises and the +# process exits, so the kubelet crashloops the pod and an e2e leg fails before it +# ever reaches training. The pod recovers minutes later and looks healthy by the +# time anyone inspects, so the only evidence is the PREVIOUS container's log — +# which is what made this expensive to diagnose. The fix is a chart-level init +# gate (the retry does not live in the jobs-manager image), and the behaviour +# that matters is the not-ready-THEN-ready handoff plus the bounded, loud failure +# when MySQL never comes up. +# +# TESTS THE REAL SCRIPT, NOT A COPY (rule 1: derive, don't restate). The wait +# loop is extracted from the RENDERED chart and executed against a fake `nc` +# (and a no-op `sleep`) on PATH, exactly as wait-for-ingest-pod.bats drives a +# fake `kubectl`. A hand-copied loop here would pass while the chart's own drifted. + +setup() { + CHART="${BATS_TEST_DIRNAME}/../../client" + VALUES="${CHART}/ci/bm-values.yaml" + require_tool helm || return 1 + require_tool python3 || return 1 + require_pymodule yaml PyYAML || return 1 + [ -d "$CHART" ] || { echo "chart directory not found at $CHART" >&2; return 1; } + [ -f "$VALUES" ] || { echo "CI values file not found at $VALUES" >&2; return 1; } + + TMP="$(mktemp -d)" + BIN="$TMP/bin"; mkdir -p "$BIN" + NCCALLS="$TMP/nc_calls"; printf '0' >"$NCCALLS" + # READY_AT: the attempt on which the fake `nc` starts succeeding. A value the + # loop can never reach (999) is "MySQL never comes up". + READY_AT="$TMP/ready_at"; printf '999' >"$READY_AT" + + # Fake `nc`: ignores every flag/host/port the script passes and answers purely + # from the counter, so a test controls exactly when the port "opens". exit 0 = + # listening, non-zero = connection refused (errno 111), which is the case. + cat >"$BIN/nc" <"$NCCALLS" +[ "\$n" -ge "\$(cat "$READY_AT")" ] && exit 0 || exit 1 +EOF + # No-op `sleep` so the bounded-failure case runs its whole cap instantly + # instead of taking the real 240s. + printf '#!/usr/bin/env bash\nexit 0\n' >"$BIN/sleep" + chmod +x "$BIN/nc" "$BIN/sleep" + + # Extract with the REAL tools on PATH (helm, python3)... + SCRIPT="$TMP/wait.sh" + extract_wait_for_mysql >"$SCRIPT" + [ -s "$SCRIPT" ] || { echo "could not extract the wait-for-mysql command from the rendered chart" >&2; return 1; } + + # ...then put the fakes ahead of the real tools before the wait loop runs. + # Without this the loop would hit real DNS/connect on a non-existent host and + # the bounded-failure case would take its full real 240s cap. + PATH="$BIN:$PATH" +} + +teardown() { [ -n "${TMP:-}" ] && rm -rf "$TMP"; } + +# A missing tool is a SKIP on a laptop and a FAILURE in CI — a required gate must +# never report green on assertions it silently skipped (see chart-pull-secret.bats). +require_tool() { + command -v "$1" >/dev/null && return 0 + if [ "${CI:-}" = "true" ]; then + echo "::error::$1 is missing in CI, so this chart-render guard would be" >&2 + echo "::error::skipped rather than run. Install it in the job instead." >&2 + return 1 + fi + skip "$1 not installed (local run)" +} + +require_pymodule() { + python3 -c "import $1" >/dev/null 2>&1 && return 0 + if [ "${CI:-}" = "true" ]; then + echo "::error::python3 module '$1' ($2) is missing in CI (pip install $2)." >&2 + return 1 + fi + skip "python3 module '$1' ($2) not installed (local run)" +} + +# Render the REAL chart and pull out the wait-for-mysql initContainer's shell +# body (command[2]). Parsed as YAML, not grepped: there are two init containers +# and two app containers, and only a structured read can pick the right one by +# name regardless of its index (it is [0] on CSI, [1] on hostPath). +extract_wait_for_mysql() { + helm template myrel "$CHART" -f "$VALUES" --namespace tracebloc \ + -s templates/jobs-manager-deployment.yaml 2>/dev/null | python3 -c ' +import sys, yaml +for doc in yaml.safe_load_all(sys.stdin.read()): + if not doc or doc.get("kind") != "Deployment": + continue + if "jobs-manager" not in doc["metadata"]["name"]: + continue + for c in (doc["spec"]["template"]["spec"].get("initContainers") or []): + if c["name"] == "wait-for-mysql": + sys.stdout.write(c["command"][2]) +' +} + +@test "hands off to the api container once MySQL starts accepting connections" { + printf '3' >"$READY_AT" # refused twice, then listening on the 3rd attempt + run sh "$SCRIPT" + [ "$status" -eq 0 ] || { echo "expected exit 0, got $status: $output" >&2; return 1; } + [[ "$output" == *"not ready yet (attempt 1/80)"* ]] || return 1 + [[ "$output" == *"is accepting connections (attempt 3/80)"* ]] || return 1 +} + +@test "the very first probe succeeding hands off immediately (no spurious wait)" { + printf '1' >"$READY_AT" + run sh "$SCRIPT" + [ "$status" -eq 0 ] || return 1 + [[ "$output" == *"is accepting connections (attempt 1/80)"* ]] || return 1 + [[ "$output" != *"not ready yet"* ]] || return 1 +} + +@test "is BOUNDED and fails LOUDLY when MySQL never comes up (not an infinite Pending)" { + # READY_AT stays 999 — the port never opens. The loop must give up at the cap + # with a non-zero exit and a diagnosable message, not spin forever. + run sh "$SCRIPT" + [ "$status" -eq 1 ] || { echo "expected exit 1, got $status" >&2; return 1; } + [[ "$output" == *"FATAL"* ]] || return 1 + [[ "$output" == *"is not listening"* ]] || return 1 + # It actually exhausted the cap rather than bailing early. + [[ "$output" == *"attempt 80/80"* ]] || return 1 +} From 0533ec4fc84ebdd0d0681bb5bac8925a8b58d005 Mon Sep 17 00:00:00 2001 From: Arturo Peroni Date: Tue, 1 Sep 2026 09:01:19 +0200 Subject: [PATCH 2/4] fix(chart): give wait-for-mysql equalised resources; sharpen the readiness rationale (backend#2913) Review (Asad on client#941): - wait-for-mysql is the FIRST init container on a CSI install (init-writable-data is hostPath-only), so leaving it unresourced silently forbade Guaranteed QoS on a profile whose api/pods-monitor requests==limits are equalised (eks-values), and collided with client#922's pod-qos goldens. Set requests == limits on both dimensions (cpu 10m / memory 16Mi): Guaranteed stays reachable, the default stays Burstable (api cpu is 250m/1000m), the pod stays admissible under a ResourceQuota on requests.cpu, and the cost is nil (k8s takes max init request vs sum of app containers; 10m << api's 250m). Guarded by an equality assertion in the unit test. - Rewrote the TCP-connect justification: the load-bearing reason is structural, not the custom image's bind timing. mysql-client is a plain selector Service with no publishNotReadyAddresses, so its EndpointSlice carries only Ready addresses -- a successful connect proves mysqld passed its mysqladmin-ping readinessProbe, and adding publishNotReadyAddresses later would downgrade the gate to a bare TCP probe. Co-Authored-By: Claude Opus 4.8 --- client/templates/jobs-manager-deployment.yaml | 31 +++++++++++++++++-- client/tests/jobs_manager_test.yaml | 15 +++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index adf85290..6144db84 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -173,9 +173,14 @@ spec: # is refused, up to ~480s if a connect hangs) sits above MySQL's own # worst-case startup — its startupProbe tolerates ~130s on a fresh datadir # (mysql-deployment.yaml: initialDelaySeconds 10 + failureThreshold 24 * - # periodSeconds 5). A TCP connect is the right signal: errno 111 is exactly - # "not listening yet", and - # mysqld binds :3306 only once it is ready to accept connections. Same + # periodSeconds 5). A TCP connect through the mysql-client Service is the + # right signal, and for a STRUCTURAL reason, not just because errno 111 means + # "not listening": mysql-client is a plain selector Service with no + # publishNotReadyAddresses, so its EndpointSlice carries ONLY Ready addresses + # (mysql-service.yaml). A successful connect therefore proves mysqld passed + # its mysqladmin-ping readinessProbe (mysql-deployment.yaml), not merely that + # a socket is open — so adding publishNotReadyAddresses to that Service later + # would quietly downgrade this gate to a bare TCP probe. Same # busybox image as init-writable-data, but non-root — it needs only the # network, no writes; busybox has no baked non-root user, so runAsUser is set # explicitly to satisfy the pod's runAsNonRoot. @@ -190,6 +195,26 @@ spec: drop: ["ALL"] seccompProfile: type: RuntimeDefault + # requests == limits on BOTH dimensions, deliberately (Saqlain/Asad on + # client#941). On CSI this is the pod's FIRST init container + # (init-writable-data is hostPath-only), so an UNRESOURCED init here would + # silently forbid Guaranteed QoS on a profile whose api/pods-monitor + # requests==limits are equalised (e.g. eks-values.yaml): the pod would + # render Burstable instead, with no error and no log line — the exact + # regression client#922's pod-qos goldens exist to catch. Equalising this + # init keeps Guaranteed reachable, leaves the default Burstable (api's cpu + # is 250m/1000m regardless), and costs effectively nothing: Kubernetes + # takes the MAX of init requests against the SUM of the app containers, and + # 10m sits far under api's 250m, so the pod's effective request does not + # move. It also keeps the pod admissible under a ResourceQuota on + # requests.cpu, which would otherwise reject it outright on CSI. + resources: + requests: + cpu: "10m" + memory: "16Mi" + limits: + cpu: "10m" + memory: "16Mi" command: - 'sh' - '-c' diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index 752050a0..2e3ddf1a 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -873,6 +873,21 @@ tests: - equal: path: spec.template.spec.initContainers[0].securityContext.capabilities.drop value: ["ALL"] + # requests == limits on BOTH dimensions (client#941): an unresourced init + # would forbid Guaranteed QoS on CSI (where this is the only init container) + # and collide with client#922's pod-qos goldens. Guard the equality directly. + - equal: + path: spec.template.spec.initContainers[0].resources.requests.cpu + value: "10m" + - equal: + path: spec.template.spec.initContainers[0].resources.limits.cpu + value: "10m" + - equal: + path: spec.template.spec.initContainers[0].resources.requests.memory + value: "16Mi" + - equal: + path: spec.template.spec.initContainers[0].resources.limits.memory + value: "16Mi" # gates on the mysql Service host:port the api container also uses - matchRegex: path: spec.template.spec.initContainers[0].command[2] From 14e47df5756eed98c5c95bf9f6bb7bb8b00c5268 Mon Sep 17 00:00:00 2001 From: Arturo Peroni Date: Tue, 1 Sep 2026 09:16:13 +0200 Subject: [PATCH 3/4] change --- ingestor/templates/job.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 ingestor/templates/job.yaml diff --git a/ingestor/templates/job.yaml b/ingestor/templates/job.yaml new file mode 100644 index 00000000..5c63f681 --- /dev/null +++ b/ingestor/templates/job.yaml @@ -0,0 +1,2 @@ +kind: Job +new: true From a0914f827a3997caa01a2d536fefcf872954bbdb Mon Sep 17 00:00:00 2001 From: Arturo Peroni Date: Tue, 1 Sep 2026 09:22:08 +0200 Subject: [PATCH 4/4] fix(chart): drop stray ingestor/templates/job.yaml swept into the merge (backend#2913) A two-line stub (kind: Job / new: true) with no apiVersion/metadata/spec was pulled into the conflict-resolution merge commit by git add -A from an untracked file in the shared worktree. Helm renders every non-_ template, so it broke the ingestor chart (Helm lint / drift / bugbot) and tripped the version-bump gate on an unbumped ingestor Chart.yaml. It has nothing to do with this PR. Remove it. Co-Authored-By: Claude Opus 4.8 --- ingestor/templates/job.yaml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 ingestor/templates/job.yaml diff --git a/ingestor/templates/job.yaml b/ingestor/templates/job.yaml deleted file mode 100644 index 5c63f681..00000000 --- a/ingestor/templates/job.yaml +++ /dev/null @@ -1,2 +0,0 @@ -kind: Job -new: true