celttechie · Nov 10, 2026
diff --git a/README.md b/README.md
@@ -59,6 +59,7 @@ uds-bundle-dev-test/
 ├── scripts/                        # Build, compliance, and chaos/tutor automation
 │   ├── common.py                   # KUBECONFIG discovery, SSH tunnel auto-recovery & runner
 │   ├── drill_manager.py            # Interactive CLI Tutor & Chaos Injection Engine
+│   ├── drills/                     # Modular YAML scenario catalog & diagnostic cheatsheet
 │   ├── build-bundle.sh             # Builds Zarf packages and creates .tar.zst bundle
 │   ├── test-deploy.sh              # Deploys the compiled UDS bundle to target cluster
 │   ├── test-compliance.sh          # Runs Lula OSCAL IL5 compliance validation

diff --git a/docs/design/platform-bundle-architecture.md b/docs/design/platform-bundle-architecture.md
@@ -40,6 +40,7 @@ uds-bundle-dev-test/
 ├── scripts/
 │   ├── common.py               # KUBECONFIG discovery, SSH tunnel auto-recovery & runner
 │   ├── drill_manager.py        # Interactive CLI Tutor & Chaos Injection Engine
+│   ├── drills/                 # Modular YAML scenario catalog & diagnostic cheatsheet
 │   ├── build-bundle.sh         # Builds individual Zarf packages + UDS bundle (.tar.zst)
 │   ├── test-deploy.sh          # Cluster deploy verification
 │   ├── test-compliance.sh      # Lula OSCAL IL5 compliance validator

diff --git a/scripts/drill_manager.py b/scripts/drill_manager.py
diff --git a/scripts/drills/cheatsheet.yaml b/scripts/drills/cheatsheet.yaml
@@ -0,0 +1,40 @@
+- title: 1. KUBERNETES WORKLOAD & NETWORK DIAGNOSTICS (kubectl)
+  commands:
+  - cmd: kubectl get pods -n ${NAMESPACE} -o wide
+    desc: Check pod status, restarts, IPs, and assigned nodes
+  - cmd: kubectl get events -n ${NAMESPACE} --sort-by=.metadata.creationTimestamp
+    desc: Chronological namespace events (probe fails, pull errors)
+  - cmd: kubectl describe pod -n ${NAMESPACE} -l app=order-api
+    desc: Inspect container exit codes (137=OOM), probes, and events
+  - cmd: kubectl logs -n ${NAMESPACE} -l app=queue-worker --previous
+    desc: View stdout/stderr from a previously crashed container
+  - cmd: kubectl get svc,ep,netpol -n ${NAMESPACE}
+    desc: Inspect Services, backing Endpoint IPs/ports, and NetworkPolicies
+  - cmd: kubectl rollout undo deployment/<name> -n ${NAMESPACE}
+    desc: Roll back a broken Deployment to its previous revision
+- title: 2. ZARF AIR-GAP REGISTRY & WEBHOOK DIAGNOSTICS (zarf)
+  commands:
+  - cmd: zarf tools registry catalog
+    desc: List all container repositories inside the in-cluster Zarf registry
+  - cmd: zarf tools registry ls library/python
+    desc: List available tags for an image in the internal Zarf registry
+  - cmd: zarf package list
+    desc: List all Zarf packages deployed in the cluster
+  - cmd: zarf package inspect <pkg.tar.zst>
+    desc: Inspect components, images, and SBOMs inside a Zarf archive
+  - cmd: kubectl logs -n zarf -l app=agent-hook --tail=30
+    desc: Inspect Zarf mutating webhook pod mutation logs
+- title: 3. UDS BUNDLE & PEPR POLICY DIAGNOSTICS (uds)
+  commands:
+  - cmd: uds inspect build/uds-bundle-*.tar.zst
+    desc: Inspect UDS bundle manifest and constituent Zarf packages
+  - cmd: uds deploy build/uds-bundle-*.tar.zst --confirm
+    desc: Deploy a compiled UDS bundle across the air-gap
+  - cmd: uds monitor
+    desc: Stream real-time UDS Pepr policy admission & network events
+- title: 4. LULA OSCAL COMPLIANCE-AS-CODE DIAGNOSTICS (lula)
+  commands:
+  - cmd: lula validate -f docs/compliance/oscal-il5.yaml -o docs/compliance/assessment-results.yaml
+    desc: Evaluate live cluster state against NIST 800-53 OSCAL controls
+  - cmd: lula evaluate -f docs/compliance/assessment-results.yaml
+    desc: Gate pass/fail status on generated OSCAL assessment results
diff --git a/scripts/drills/comp-oom-killed.yaml b/scripts/drills/comp-oom-killed.yaml
@@ -0,0 +1,37 @@
+id: comp-oom-killed
+domain: Compute
+tools:
+- kubectl
+title: Out Of Memory CrashLoop (OOMKilled / Exit Code 137)
+description: Injects an unbounded memory allocation loop into `queue-worker` in '${NAMESPACE}'.
+symptoms: 'queue-worker restarts repeatedly in CrashLoopBackOff with Last State: OOMKilled (Exit Code 137).'
+inject_cmds:
+- |-
+  cat << 'EOF' | kubectl patch deployment queue-worker -n ${NAMESPACE} --type='json' --patch-file /dev/stdin
+    [{"op":"replace","path":"/spec/template/spec/containers/0/command","value":["/bin/sh","-c","python3 -c \"import time; b=[]; print('Leaking memory...'); [b.append('X'*1024*1024) or time.sleep(0.05) for _ in range(500)]\""]}]
+    EOF
+heal_cmds:
+- kubectl rollout undo deployment/queue-worker -n ${NAMESPACE}
+verify_cmd: kubectl get pods -n ${NAMESPACE} -l app=queue-worker
+tutor_steps:
+- phase: 'Phase 1: Detect (Identify Crashing Pod)'
+  prompt: List all pods in '${NAMESPACE}' to find the crashing workload.
+  recommended_cmd: kubectl get pods -n ${NAMESPACE}
+  hint: Run `kubectl get pods -n ${NAMESPACE}`.
+  explanation: '`queue-worker` has multiple restarts and is entering `OOMKilled` / `CrashLoopBackOff`.'
+- phase: 'Phase 2: Isolate (Check Container Exit Code & Last Termination State)'
+  prompt: Describe the `queue-worker` pod to inspect `Last State`, `Reason`, and `Exit Code`.
+  recommended_cmd: kubectl describe pod -n ${NAMESPACE} -l app=queue-worker | grep -A 12 'Last State:'
+  hint: Run `kubectl describe pod -n ${NAMESPACE} -l app=queue-worker`.
+  explanation: '`Reason: OOMKilled` and `Exit Code: 137` (SIGKILL = 128 + 9) prove the Linux kernel cgroup OOM killer terminated
+    the container for exceeding its 128Mi memory limit.'
+- phase: 'Phase 3: Root Cause (Inspect Previous Container Logs)'
+  prompt: Fetch logs from the previous crashed container instance of `queue-worker`.
+  recommended_cmd: kubectl logs -n ${NAMESPACE} -l app=queue-worker --previous
+  hint: Run `kubectl logs -n ${NAMESPACE} -l app=queue-worker --previous`.
+  explanation: The logs show `Leaking memory...` right before the kernel killed the process.
+- phase: 'Phase 4: Remediate & Verify (Roll Back Deployment)'
+  prompt: Roll back the `queue-worker` deployment in '${NAMESPACE}' to its previous healthy revision.
+  recommended_cmd: kubectl rollout undo deployment/queue-worker -n ${NAMESPACE}
+  hint: Run `kubectl rollout undo deployment/queue-worker -n ${NAMESPACE}`.
+  explanation: '`kubectl rollout undo` restores the healthy worker loop and stabilizes memory usage.'
diff --git a/scripts/drills/comp-probe-fail.yaml b/scripts/drills/comp-probe-fail.yaml
@@ -0,0 +1,33 @@
+id: comp-probe-fail
+domain: Compute
+tools:
+- kubectl
+title: Liveness Probe HTTP 404 Failure Loop
+description: Configures an invalid livenessProbe path (`/healthz-broken-path`) on `order-api` in '${NAMESPACE}'.
+symptoms: order-api containers are repeatedly killed and restarted by kubelet due to HTTP 404 liveness probe failures.
+inject_cmds:
+- kubectl patch deployment order-api -n ${NAMESPACE} --type='json' -p='[{"op":"replace","path":"/spec/template/spec/containers/0/livenessProbe/httpGet/path","value":"/healthz-broken-path"}]'
+heal_cmds:
+- kubectl patch deployment order-api -n ${NAMESPACE} --type='json' -p='[{"op":"replace","path":"/spec/template/spec/containers/0/livenessProbe/httpGet/path","value":"/healthz"}]'
+verify_cmd: kubectl get events -n ${NAMESPACE} --sort-by=.metadata.creationTimestamp | tail -n 10
+tutor_steps:
+- phase: 'Phase 1: Detect (Observe Pod Restarts)'
+  prompt: Check pod statuses and restart counts in '${NAMESPACE}'.
+  recommended_cmd: kubectl get pods -n ${NAMESPACE} -l app=order-api
+  hint: Run `kubectl get pods -n ${NAMESPACE} -l app=order-api`.
+  explanation: '`order-api` pods are restarting every ~30 seconds.'
+- phase: 'Phase 2: Isolate (Inspect Namespace Events)'
+  prompt: Check recent Kubernetes events in '${NAMESPACE}' sorted by creation timestamp.
+  recommended_cmd: kubectl get events -n ${NAMESPACE} --sort-by=.metadata.creationTimestamp | tail -n 15
+  hint: Run `kubectl get events -n ${NAMESPACE} --sort-by=.metadata.creationTimestamp`.
+  explanation: 'Events show `Warning Unhealthy: Liveness probe failed: HTTP probe failed with statuscode: 404`.'
+- phase: 'Phase 3: Root Cause (Compare Liveness vs Readiness Probe Paths)'
+  prompt: Inspect the `livenessProbe` and `readinessProbe` definitions on `order-api`.
+  recommended_cmd: kubectl describe deploy order-api -n ${NAMESPACE} | grep -E 'Liveness:|Readiness:'
+  hint: Run `kubectl describe deploy order-api -n ${NAMESPACE}`.
+  explanation: Readiness checks `/healthz` (200 OK), while Liveness was misconfigured to `/healthz-broken-path` (404).
+- phase: 'Phase 4: Remediate & Verify (Fix Liveness Probe Path)'
+  prompt: Patch the `order-api` livenessProbe path back to `/healthz`.
+  recommended_cmd: kubectl patch deployment order-api -n ${NAMESPACE} --type='json' -p='[{"op":"replace","path":"/spec/template/spec/containers/0/livenessProbe/httpGet/path","value":"/healthz"}]'
+  hint: Run `kubectl rollout undo deployment/order-api -n ${NAMESPACE}` or patch `/healthz`.
+  explanation: Liveness probes now return HTTP 200 and pod restarts cease.
diff --git a/scripts/drills/comp-unschedulable.yaml b/scripts/drills/comp-unschedulable.yaml
@@ -0,0 +1,34 @@
+id: comp-unschedulable
+domain: Compute
+tools:
+- kubectl
+title: Unschedulable Pod / NodeSelector Mismatch
+description: 'Applies a non-existent `nodeSelector` (`topology.kubernetes.io/zone: non-existent-zone-c`) to `order-api` in
+  ''${NAMESPACE}''.'
+symptoms: New order-api pods remain stuck in Pending state with a FailedScheduling event.
+inject_cmds:
+- kubectl patch deployment order-api -n ${NAMESPACE} --type='json' -p='[{"op":"add","path":"/spec/template/spec/nodeSelector","value":{"topology.kubernetes.io/zone":"non-existent-zone-c"}}]'
+heal_cmds:
+- kubectl patch deployment order-api -n ${NAMESPACE} --type='json' -p='[{"op":"remove","path":"/spec/template/spec/nodeSelector"}]'
+verify_cmd: kubectl get pods -n ${NAMESPACE} -l app=order-api
+tutor_steps:
+- phase: 'Phase 1: Detect (Spot Pending Pods)'
+  prompt: Check pod statuses in '${NAMESPACE}'.
+  recommended_cmd: kubectl get pods -n ${NAMESPACE} -o wide
+  hint: Run `kubectl get pods -n ${NAMESPACE} -o wide`.
+  explanation: New `order-api` replica is stuck in `Pending` with `<none>` assigned for `NODE`.
+- phase: 'Phase 2: Isolate (Check Scheduler Failure Event)'
+  prompt: Describe the pending `order-api` pod to see why `default-scheduler` rejected it.
+  recommended_cmd: kubectl describe pod -n ${NAMESPACE} -l app=order-api | tail -n 15
+  hint: Run `kubectl describe pod -n ${NAMESPACE} -l app=order-api`.
+  explanation: '`Warning FailedScheduling: 0/1 nodes are available: 1 node(s) didn''t match Pod''s node affinity/selector`.'
+- phase: 'Phase 3: Root Cause (Compare Node Labels vs Pod NodeSelector)'
+  prompt: Check the labels on your cluster nodes and compare with the deployment's `nodeSelector`.
+  recommended_cmd: kubectl get nodes --show-labels
+  hint: Run `kubectl get nodes --show-labels`.
+  explanation: The deployment requires `topology.kubernetes.io/zone=non-existent-zone-c`, which no node has.
+- phase: 'Phase 4: Remediate & Verify (Remove Invalid NodeSelector)'
+  prompt: Remove the invalid `nodeSelector` from `order-api` in '${NAMESPACE}'.
+  recommended_cmd: kubectl patch deployment order-api -n ${NAMESPACE} --type='json' -p='[{"op":"remove","path":"/spec/template/spec/nodeSelector"}]'
+  hint: Run `kubectl patch deployment order-api -n ${NAMESPACE} --type='json' -p='[{"op":"remove","path":"/spec/template/spec/nodeSelector"}]'`.
+  explanation: The scheduler immediately binds the pod to the node.
diff --git a/scripts/drills/net-policy-block.yaml b/scripts/drills/net-policy-block.yaml
@@ -0,0 +1,55 @@
+id: net-policy-block
+domain: UDS/Networking
+tools:
+- kubectl
+- uds
+title: Zero-Trust NetworkPolicy Ingress Block
+description: 'Applies an overriding NetworkPolicy (''deny-order-api-ingress'') in ''${NAMESPACE}'' that selects `app: order-api`
+  and blocks all ingress traffic from `frontend`.'
+symptoms: All pods show 1/1 Running, but requests from frontend to order-api:8080 time out due to a restrictive NetworkPolicy.
+inject_cmds:
+- |-
+  cat << 'EOF' | kubectl apply -f -
+    apiVersion: networking.k8s.io/v1
+    kind: NetworkPolicy
+    metadata:
+      name: deny-order-api-ingress
+      namespace: ${NAMESPACE}
+    spec:
+      podSelector:
+        matchLabels:
+          app: order-api
+      policyTypes:
+        - Ingress
+      ingress:
+        - from:
+            - podSelector:
+                matchLabels:
+                  app: nonexistent-service
+    EOF
+heal_cmds:
+- kubectl delete networkpolicy deny-order-api-ingress -n ${NAMESPACE} --ignore-not-found
+verify_cmd: kubectl get networkpolicy -n ${NAMESPACE}
+tutor_steps:
+- phase: 'Phase 1: Detect (Pods Running, Traffic Timing Out)'
+  prompt: Verify that pods and services in '${NAMESPACE}' appear healthy despite the connectivity failure.
+  recommended_cmd: kubectl get pods,svc -n ${NAMESPACE}
+  hint: Run `kubectl get pods,svc -n ${NAMESPACE}`.
+  explanation: Both `frontend` and `order-api` pods are `Running` and Services have valid ClusterIPs.
+- phase: 'Phase 2: Isolate (List Active NetworkPolicies)'
+  prompt: List the Kubernetes NetworkPolicies active in the '${NAMESPACE}' namespace.
+  recommended_cmd: kubectl get networkpolicy -n ${NAMESPACE}
+  hint: Run `kubectl get netpol -n ${NAMESPACE}`.
+  explanation: Alongside `allow-training-internal`, there is a suspicious policy named `deny-order-api-ingress` targeting
+    `app=order-api`.
+- phase: 'Phase 3: Root Cause (Inspect NetworkPolicy Rules)'
+  prompt: Describe `deny-order-api-ingress` to see which pods are allowed to connect to `order-api`.
+  recommended_cmd: kubectl describe networkpolicy deny-order-api-ingress -n ${NAMESPACE}
+  hint: Run `kubectl describe netpol deny-order-api-ingress -n ${NAMESPACE}`.
+  explanation: The policy restricts ingress on `app=order-api` exclusivamente to `app=nonexistent-service`, dropping packets
+    from `frontend`.
+- phase: 'Phase 4: Remediate & Verify (Remove Rogue NetworkPolicy)'
+  prompt: Delete the rogue `deny-order-api-ingress` NetworkPolicy in '${NAMESPACE}'.
+  recommended_cmd: kubectl delete networkpolicy deny-order-api-ingress -n ${NAMESPACE}
+  hint: Run `kubectl delete netpol deny-order-api-ingress -n ${NAMESPACE}`.
+  explanation: Deleting the rogue policy restores inter-service communication under `allow-training-internal`.
diff --git a/scripts/drills/net-port-mismatch.yaml b/scripts/drills/net-port-mismatch.yaml
@@ -0,0 +1,35 @@
+id: net-port-mismatch
+domain: UDS/Networking
+tools:
+- kubectl
+title: Service TargetPort Misconfiguration (Connection Refused)
+description: Changes `order-api` Service targetPort in '${NAMESPACE}' from 8080 to 9090.
+symptoms: order-api pods are 1/1 Running and DNS resolves, but connections to order-api:8080 immediately fail with 'Connection
+  refused'.
+inject_cmds:
+- kubectl patch svc order-api -n ${NAMESPACE} --type='json' -p='[{"op":"replace","path":"/spec/ports/0/targetPort","value":9090}]'
+heal_cmds:
+- kubectl patch svc order-api -n ${NAMESPACE} --type='json' -p='[{"op":"replace","path":"/spec/ports/0/targetPort","value":8080}]'
+verify_cmd: kubectl get svc,ep order-api -n ${NAMESPACE}
+tutor_steps:
+- phase: 'Phase 1: Detect (Inspect Service & Endpoints)'
+  prompt: Check the `order-api` Service and Endpoints in '${NAMESPACE}'.
+  recommended_cmd: kubectl get svc,ep order-api -n ${NAMESPACE}
+  hint: Run `kubectl get svc,ep order-api -n ${NAMESPACE}`.
+  explanation: 'Look closely at the `ENDPOINTS` column: the pod IPs are listed with port `:9090` instead of `:8080`!'
+- phase: 'Phase 2: Isolate (Compare Service TargetPort vs ContainerPort)'
+  prompt: Describe the `order-api` Service and check the container port on the `order-api` Deployment.
+  recommended_cmd: kubectl describe svc order-api -n ${NAMESPACE}
+  hint: Run `kubectl describe svc order-api -n ${NAMESPACE}`.
+  explanation: '`Port` is `8080/TCP`, but `TargetPort` was changed to `9090/TCP`, while the Python HTTP server inside the
+    container listens on port `8080`.'
+- phase: 'Phase 3: Root Cause (Verify Container Listening Port)'
+  prompt: Check the containerPort configured on the `order-api` deployment.
+  recommended_cmd: kubectl get deploy order-api -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].ports}'
+  hint: Run `kubectl get deploy order-api -n ${NAMESPACE} -o jsonpath='{.spec.template.spec.containers[0].ports}'`.
+  explanation: 'The container exposes `containerPort: 8080`, confirming the Service `targetPort: 9090` is wrong.'
+- phase: 'Phase 4: Remediate & Verify (Patch Service TargetPort to 8080)'
+  prompt: Patch the `order-api` Service in '${NAMESPACE}' so `targetPort` is `8080`.
+  recommended_cmd: kubectl patch svc order-api -n ${NAMESPACE} --type='json' -p='[{"op":"replace","path":"/spec/ports/0/targetPort","value":8080}]'
+  hint: Run `kubectl patch svc order-api -n ${NAMESPACE} -p '{"spec":{"ports":[{"port":8080,"targetPort":8080}]}}'`.
+  explanation: Endpoints immediately update to `:8080` and traffic flows normally.
diff --git a/scripts/drills/net-service-endpoint.yaml b/scripts/drills/net-service-endpoint.yaml
@@ -0,0 +1,33 @@
+id: net-service-endpoint
+domain: UDS/Networking
+tools:
+- kubectl
+title: Service Selector Label Mismatch (Empty Endpoints)
+description: 'Changes `order-api` Service selector in ''${NAMESPACE}'' to `app: order-api-wrong-label`.'
+symptoms: Service `order-api` has `<none>` in its Endpoints list (`kubectl get ep order-api -n ${NAMESPACE}`).
+inject_cmds:
+- kubectl patch svc order-api -n ${NAMESPACE} --type='json' -p='[{"op":"replace","path":"/spec/selector/app","value":"order-api-wrong-label"}]'
+heal_cmds:
+- kubectl patch svc order-api -n ${NAMESPACE} --type='json' -p='[{"op":"replace","path":"/spec/selector/app","value":"order-api"}]'
+verify_cmd: kubectl get ep order-api -n ${NAMESPACE}
+tutor_steps:
+- phase: 'Phase 1: Detect (Check Service Endpoints)'
+  prompt: Check the Endpoints object for `order-api` in '${NAMESPACE}'.
+  recommended_cmd: kubectl get endpoints order-api -n ${NAMESPACE}
+  hint: Run `kubectl get ep order-api -n ${NAMESPACE}`.
+  explanation: The `ENDPOINTS` column is `<none>`, meaning the Service has zero backing pods.
+- phase: 'Phase 2: Isolate (Inspect Service Selector)'
+  prompt: Describe the `order-api` Service to see what pod labels its `Selector` is looking for.
+  recommended_cmd: kubectl describe svc order-api -n ${NAMESPACE}
+  hint: Run `kubectl describe svc order-api -n ${NAMESPACE}` and check the `Selector:` line.
+  explanation: The Service selector is looking for `app=order-api-wrong-label`.
+- phase: 'Phase 3: Root Cause (Compare Against Actual Pod Labels)'
+  prompt: List the pods in '${NAMESPACE}' with their labels displayed.
+  recommended_cmd: kubectl get pods -n ${NAMESPACE} --show-labels
+  hint: Run `kubectl get pods -n ${NAMESPACE} --show-labels`.
+  explanation: The running pods have label `app=order-api`, not `app=order-api-wrong-label`.
+- phase: 'Phase 4: Remediate & Verify (Fix Service Selector)'
+  prompt: 'Patch the `order-api` Service selector back to `app: order-api`.'
+  recommended_cmd: kubectl patch svc order-api -n ${NAMESPACE} -p '{"spec":{"selector":{"app":"order-api"}}}'
+  hint: Run `kubectl patch svc order-api -n ${NAMESPACE} -p '{"spec":{"selector":{"app":"order-api"}}}'`.
+  explanation: Kubernetes immediately repopulates the `Endpoints` slice with the running pod IPs.