From a645c5440e23fbfc8a22bb40841d6fda850bd818 Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:00:34 +0000 Subject: [PATCH 01/12] Collect the BlueField SNAP-4 container version and grade it per release train SNAP-4 runs on the DPU Arm cores, so the virtio-net check reads it on the same DPU path: locally on a DPU, otherwise one RShim SSH command that lists the running containers (crictl, retried under sudo -n) and the kubelet manifests. The read never pulls, starts, stops, or execs into a container. security_version_audit.py grades each reading against its own train in snap4Container (4.5 -> 4.5.5, 4.9 -> 4.9.1). A version between or below the trains moves to the next fix above it, and one newer than every fix passes. A below-minimum reading fails whatever the coverage; passing readings clear the cluster only when every DPU was read or cannot run SNAP-4 now. The result is a new snap4-container check in cmax audit security and a securityVersions.snap4Container criterion in the full audit. --- README.md | 1 + README_zh.md | 1 + cmax/audit_profiles.py | 1 + cmax/audit_report.py | 8 + cmax/criteria_links.py | 1 + cmax/minimum_links.py | 2 + cmax/scripts/1-audit/audit-common.sh | 12 +- cmax/scripts/1-audit/audit_findings.py | 19 + .../1-audit/checks/fabric/virtio-net-check.py | 374 +++++++++++++++-- cmax/scripts/1-audit/security_fleet.py | 6 +- .../scripts/1-audit/security_version_audit.py | 197 +++++++++ cmax/security.py | 78 ++++ tests/audit/test_security_cli.py | 19 +- tests/audit/test_snap4_container.py | 389 ++++++++++++++++++ 14 files changed, 1072 insertions(+), 36 deletions(-) create mode 100644 tests/audit/test_snap4_container.py diff --git a/README.md b/README.md index e5668ea..c55630d 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,7 @@ audit_profile: - securityVersions.connectxFirmware.status - securityVersions.dcgm.status - securityVersions.dcgmExporter.status + - securityVersions.snap4Container.status - securityVersions.virtioNetBluefield.status - securityVersions.virtioNetBluefield.exposure - securityVersions.dpuHostIsolation.status diff --git a/README_zh.md b/README_zh.md index 3add6ec..981b7d6 100644 --- a/README_zh.md +++ b/README_zh.md @@ -129,6 +129,7 @@ audit_profile: - securityVersions.connectxFirmware.status - securityVersions.dcgm.status - securityVersions.dcgmExporter.status + - securityVersions.snap4Container.status - securityVersions.virtioNetBluefield.status - securityVersions.virtioNetBluefield.exposure - securityVersions.dpuHostIsolation.status diff --git a/cmax/audit_profiles.py b/cmax/audit_profiles.py index 9e28e76..8fc0f4c 100644 --- a/cmax/audit_profiles.py +++ b/cmax/audit_profiles.py @@ -32,6 +32,7 @@ def includes(self, key: str) -> bool: "securityVersions.dcgm.status", "securityVersions.dcgmExporter.status", "securityVersions.virtioNetBluefield.status", + "securityVersions.snap4Container.status", } ), ), diff --git a/cmax/audit_report.py b/cmax/audit_report.py index 8b6244b..f876658 100644 --- a/cmax/audit_report.py +++ b/cmax/audit_report.py @@ -112,6 +112,9 @@ def _harness_not_applicable(harness: str) -> str: "securityVersions.virtioNetBluefield.status": ( "The BlueField VIRTIO-Net controller meets the published security minimum." ), + "securityVersions.snap4Container.status": ( + "The BlueField SNAP-4 container meets the published security minimum for its release train." + ), "securityVersions.dpuHostIsolation.status": ( "The host cannot reach the BlueField DPU control plane through the checked interfaces." ), @@ -373,6 +376,11 @@ def _title(key: str) -> str: "custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name," "IMAGES:.spec.containers[*].image' | grep -i dcgm-exporter`" ), + "securityVersions.snap4Container.status": ( + "on each BlueField-3 DPU in DPU mode (locally or over RShim with " + "`ssh ubuntu@192.168.100.2`): `sudo crictl ps` and `sudo crictl images " + "| grep doca_snap`, or `grep -i 'image:.*doca_snap' /etc/kubelet.d/*`" + ), "securityVersions.dpuHostIsolation.status": ( "on every BlueField GPU worker: read `INTERNAL_CPU_RSHIM` with " "`sudo -n mlxconfig -d q`, inspect `/dev/rshim*`, and " diff --git a/cmax/criteria_links.py b/cmax/criteria_links.py index a19b76f..bfeefda 100644 --- a/cmax/criteria_links.py +++ b/cmax/criteria_links.py @@ -93,6 +93,7 @@ "securityVersions.connectxFirmware.status": _SECURITY_PATCHING, "securityVersions.dcgm.status": _SECURITY_PATCHING, "securityVersions.dcgmExporter.status": _SECURITY_PATCHING, + "securityVersions.snap4Container.status": _SECURITY_PATCHING, "securityVersions.virtioNetBluefield.status": _SECURITY_PATCHING, "securityVersions.virtioNetBluefield.exposure": _SECURITY_ISOLATION, "securityVersions.dpuHostIsolation.status": _SECURITY_ISOLATION, diff --git a/cmax/minimum_links.py b/cmax/minimum_links.py index 25b0740..8bef0a6 100644 --- a/cmax/minimum_links.py +++ b/cmax/minimum_links.py @@ -21,6 +21,7 @@ "rocm", "dcgm", "dcgmExporter", + "snap4Container", "ubuntuNoble", } ) @@ -33,6 +34,7 @@ "docker": "docker", "connectx-firmware": "connectxFirmware", "virtio-net-bluefield": "virtioNetBluefield", + "snap4-container": "snap4Container", "fragnesia": "ubuntuNoble", } diff --git a/cmax/scripts/1-audit/audit-common.sh b/cmax/scripts/1-audit/audit-common.sh index 5fa17f5..b3bd12c 100644 --- a/cmax/scripts/1-audit/audit-common.sh +++ b/cmax/scripts/1-audit/audit-common.sh @@ -453,7 +453,7 @@ version_meets_minimum() { # which grades as a warning, instead of "fail", which grades as critical: a # machine proven vulnerable would be softened because a different machine was # unreadable. -VIRTIO_NET_SUMMARY_KEYS="virtioNet virtioNetLine virtioNetMode virtioNetSource virtioNetReason dpuIsolationJson state virtioNetWorstObserved virtioNetWorstObservedLine virtioNetWorstObservedMode virtioNetWorstObservedHost virtioNetObservedJson" +VIRTIO_NET_SUMMARY_KEYS="virtioNet virtioNetLine virtioNetMode virtioNetSource virtioNetReason dpuIsolationJson state virtioNetWorstObserved virtioNetWorstObservedLine virtioNetWorstObservedMode virtioNetWorstObservedHost virtioNetObservedJson snap4Json" # virtio_net_check_facts - resolve the BlueField VIRTIO-Net controller state. # @@ -464,6 +464,8 @@ VIRTIO_NET_SUMMARY_KEYS="virtioNet virtioNetLine virtioNetMode virtioNetSource v # VIRTIO_NET_SOURCE --virtio-net-source (how the version was read) # VIRTIO_NET_REASON --virtio-net-reason (why it was not read) # VIRTIO_NET_ISOLATION_JSON --dpu-isolation-json (the RShim posture evidence) +# SNAP4_JSON --snap4-json (the SNAP-4 container rollup, +# read on the same DPU path) # plus VIRTIO_NET_STATE for the operator log. # # The check self-dispatches per node, because a DPU is per node and a head-node @@ -490,6 +492,7 @@ virtio_net_check_facts() { VIRTIO_NET_WORST_MODE="" VIRTIO_NET_WORST_HOST="" VIRTIO_NET_OBSERVED_JSON="" + SNAP4_JSON="" local check="${WORKLOAD_DIR:-$AUDIT_COMMON_DIR}/checks/fabric/virtio-net-check.py" [[ -f "$check" ]] || check="${AUDIT_COMMON_DIR}/checks/fabric/virtio-net-check.py" @@ -542,6 +545,7 @@ for key in sys.argv[1:]: 9) VIRTIO_NET_WORST_MODE="$line" ;; 10) VIRTIO_NET_WORST_HOST="$line" ;; 11) VIRTIO_NET_OBSERVED_JSON="$line" ;; + 12) SNAP4_JSON="$line" ;; esac index=$((index + 1)) done <<< "$parsed" @@ -688,6 +692,7 @@ build_security_version_audit() { VIRTIO_NET_WORST_HOST="" VIRTIO_NET_OBSERVED_JSON="" VIRTIO_NET_ISOLATION_JSON="" + SNAP4_JSON='{"state":"not_applicable","reason":"scale-out checks do not apply to standalone","observed":[]}' else virtio_net_check_facts fi @@ -713,9 +718,10 @@ build_security_version_audit() { && args+=(--virtio-net-observed-json "$VIRTIO_NET_OBSERVED_JSON") [[ -n "$VIRTIO_NET_ISOLATION_JSON" ]] \ && args+=(--dpu-isolation-json "$VIRTIO_NET_ISOLATION_JSON") + [[ -n "$SNAP4_JSON" ]] && args+=(--snap4-json "$SNAP4_JSON") if [[ ! -f "$policy" ]] || ! command -v python3 >/dev/null 2>&1; then - printf '%s\n' '{"amdDriver":{"status":"unknown"},"nvidiaDriver":{"status":"unknown"},"nvidiaContainerToolkit":{"status":"unknown"},"cudaToolkit":{"status":"unknown"},"docker":{"status":"unknown"},"runc":{"status":"unknown"},"connectxFirmware":{"status":"unknown"},"dcgm":{"status":"unknown"},"dcgmExporter":{"status":"unknown"},"virtioNetBluefield":{"status":"unknown"},"dpuHostIsolation":{"status":"unknown"}}' + printf '%s\n' '{"amdDriver":{"status":"unknown"},"nvidiaDriver":{"status":"unknown"},"nvidiaContainerToolkit":{"status":"unknown"},"cudaToolkit":{"status":"unknown"},"docker":{"status":"unknown"},"runc":{"status":"unknown"},"connectxFirmware":{"status":"unknown"},"dcgm":{"status":"unknown"},"dcgmExporter":{"status":"unknown"},"virtioNetBluefield":{"status":"unknown"},"dpuHostIsolation":{"status":"unknown"},"snap4Container":{"status":"unknown"}}' return 0 fi @@ -760,7 +766,7 @@ build_security_version_audit() { done <<< "$check_output" python3 "$policy" "${args[@]}" 2>/dev/null \ - || printf '%s\n' '{"amdDriver":{"status":"unknown"},"nvidiaDriver":{"status":"unknown"},"nvidiaContainerToolkit":{"status":"unknown"},"cudaToolkit":{"status":"unknown"},"docker":{"status":"unknown"},"runc":{"status":"unknown"},"connectxFirmware":{"status":"unknown"},"dcgm":{"status":"unknown"},"dcgmExporter":{"status":"unknown"},"virtioNetBluefield":{"status":"unknown"},"dpuHostIsolation":{"status":"unknown"}}' + || printf '%s\n' '{"amdDriver":{"status":"unknown"},"nvidiaDriver":{"status":"unknown"},"nvidiaContainerToolkit":{"status":"unknown"},"cudaToolkit":{"status":"unknown"},"docker":{"status":"unknown"},"runc":{"status":"unknown"},"connectxFirmware":{"status":"unknown"},"dcgm":{"status":"unknown"},"dcgmExporter":{"status":"unknown"},"virtioNetBluefield":{"status":"unknown"},"dpuHostIsolation":{"status":"unknown"},"snap4Container":{"status":"unknown"}}' } amd_gpu_check_present() { diff --git a/cmax/scripts/1-audit/audit_findings.py b/cmax/scripts/1-audit/audit_findings.py index c633db0..b000c4c 100644 --- a/cmax/scripts/1-audit/audit_findings.py +++ b/cmax/scripts/1-audit/audit_findings.py @@ -751,6 +751,22 @@ def _tenant_pod_guidance( CONFIG, _status_is("unknown"), ), + # SNAP-4 runs on the DPU Arm side, so it is graded only where the DPU is + # part of the audited fabric, the same scope as the virtio-net controller. + Rule( + "securityVersions.snap4Container.status", + "BlueField SNAP-4 container below security minimum", + VERSION, + _status_is("fail"), + harnesses=SCALE_OUT_HARNESSES, + ), + Rule( + "securityVersions.snap4Container.status", + "BlueField SNAP-4 container version requires provider attestation", + CONFIG, + _status_is("unknown"), + harnesses=SCALE_OUT_HARNESSES, + ), # --- BlueField DPU ---------------------------------------------------- # The controller firmware check has four reportable states, not two. A # below-minimum controller in DPU mode is running and fails. The same firmware @@ -1640,6 +1656,8 @@ def classify_suppression(key: str, audit: dict[str, Any]) -> tuple[str, str]: "securityVersions.connectxFirmware.status": "security-connectx-firmware", "securityVersions.dcgm.status": None, "securityVersions.dcgmExporter.status": None, + # CLI-only until the criteria catalog has a SNAP-4 criterion. + "securityVersions.snap4Container.status": None, "securityVersions.virtioNetBluefield.status": "security-virtio-net-bluefield", "securityVersions.virtioNetBluefield.exposure": "security-virtio-net-bluefield", "securityVersions.dpuHostIsolation.status": "security-dpu-host-isolation", @@ -1738,6 +1756,7 @@ def _audit_data(values: dict[str, Any]) -> dict[str, Any]: "nvidiaContainerToolkit": "NVIDIA Container Toolkit", "nvidiaDriver": "NVIDIA driver", "runc": "runc", + "snap4Container": "BlueField SNAP-4 container", "virtioNetBluefield": "BlueField VIRTIO-Net controller", } diff --git a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py index 5f12b79..217802d 100755 --- a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py +++ b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py @@ -163,15 +163,16 @@ # sudo -n mst start, sudo -n mlxconfig q 4 x 15 = 60 s # local `virtnet version` + `virtnet -v` 2 x 15 = 30 s # RShim SSH: version, -v, list, mlnx-release 4 x 16 = 64 s +# SNAP-4 listing, one RShim SSH command 1 x 16 = 16 s # ------ -# 169 s +# 185 s # # The local-virtnet-succeeded path and the RShim SSH path are mutually # exclusive (collect_version only reaches SSH when the local read produced no -# version), and the succeeded path is cheaper: 15 + 60 + 15 + 2 x 15 = 120 s. -# So 169 s is a true ceiling, not a sum of alternatives. Allow 60 s for srun -# dispatch and worker startup, giving 229 s against a 300 s budget, about 24% -# margin. +# version), and the succeeded path is cheaper: 15 + 60 + 15 + 2 x 15 + 15 = +# 135 s. So 185 s is a true ceiling, not a sum of alternatives. Allow 60 s for +# srun dispatch and worker startup, giving 245 s against a 300 s budget, about +# 18% margin. # # Redo this arithmetic when changing TOOL_TIMEOUT_S or SSH_TIMEOUT_S, or when # adding a command to gather(). An overrun is no longer a false clean, because @@ -813,6 +814,31 @@ def collect_line( return line, evidence +def dpu_ssh_prefix(env: dict[str, str] = os.environ) -> list[str]: + """The RShim SSH argv every DPU-side read starts with. + + BatchMode=yes cannot prompt for a password or a passphrase, so this either + works with an already-trusted key or fails immediately. Nothing is ever + written to the RShim console and no interactive session is opened. + UserKnownHostsFile=/dev/null keeps the read-only audit from leaving a host + key behind in the operator's known_hosts, which StrictHostKeyChecking=no + would otherwise write on the first run. + """ + target = env.get("CLUSTERMAX_DPU_SSH_TARGET") or "ubuntu@192.168.100.2" + return [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + f"ConnectTimeout={SSH_TIMEOUT_S}", + target, + ] + + def collect_version( rshim: dict[str, Any], *, @@ -849,26 +875,7 @@ def empty(cli: str | None) -> dict[str, Any]: if not (rshim.get("rshimDeviceNode") or rshim.get("tmfifoNet0")): return empty(local_cli) - target = env.get("CLUSTERMAX_DPU_SSH_TARGET") or "ubuntu@192.168.100.2" - # BatchMode=yes cannot prompt for a password or a passphrase, so this either - # works with an already-trusted key or fails immediately. Nothing is ever - # written to the RShim console and no interactive session is opened. - # UserKnownHostsFile=/dev/null keeps the read-only audit from leaving a host - # key behind in the operator's known_hosts, which StrictHostKeyChecking=no - # would otherwise write on the first run. - ssh = [ - "ssh", - "-o", - "BatchMode=yes", - "-o", - "StrictHostKeyChecking=no", - "-o", - "UserKnownHostsFile=/dev/null", - "-o", - f"ConnectTimeout={SSH_TIMEOUT_S}", - target, - "virtnet", - ] + ssh = [*dpu_ssh_prefix(env), "virtnet"] remote = read_virtnet(ssh, runner=runner, timeout=SSH_TIMEOUT_S * 2) if remote is None: rshim["dpuReachedFromHost"] = False @@ -887,6 +894,298 @@ def empty(cli: str | None) -> dict[str, Any]: } +# --------------------------------------------------------------------------- +# SNAP-4 container (NVIDIA bulletin a_id 5744) +# --------------------------------------------------------------------------- +# +# DOCA SNAP-4 runs as a container on the BlueField-3 Arm cores, started by the +# DPU's kubelet from a static pod manifest in /etc/kubelet.d. Its version is the +# image tag (nvcr.io/nvidia/doca/doca_snap:4.9.1-doca3.2.0). Nothing on the host +# side carries it, so it is read on the same DPU access path as the virtio-net +# controller: locally when this check already runs on the DPU, otherwise over +# the RShim SSH link. One remote command does the whole read, so an unreachable +# DPU costs one SSH timeout. +# +# The read only lists: it never pulls an image, starts or stops a container, or +# execs into one. `crictl` needs root on a stock DPU image, so it is retried +# under `sudo -n`, which never prompts. + +SNAP4_STATE_NOT_INSTALLED = "not_installed" + +# Running containers first, then the kubelet manifests. kubelet keeps a static +# pod running for as long as its manifest exists, so a manifest is the version +# that runs whenever the container is restarted. +SNAP4_DPU_SCRIPT = r""" +echo @@SNAP4-MANIFESTS@@ +for f in /etc/kubelet.d/*; do + [ -f "$f" ] && [ -r "$f" ] && grep -i 'image:.*doca_snap' "$f" 2>/dev/null +done +if ! command -v crictl >/dev/null 2>&1; then + echo @@SNAP4-NO-CRICTL@@ + exit 0 +fi +echo @@SNAP4-PS@@ +crictl ps -o json 2>/dev/null || sudo -n crictl ps -o json 2>/dev/null || echo @@SNAP4-PS-FAILED@@ +# The bare echoes keep a marker on its own line when the JSON has no trailing +# newline. +echo +echo @@SNAP4-IMAGES@@ +crictl images -o json 2>/dev/null || sudo -n crictl images -o json 2>/dev/null +echo +echo @@SNAP4-END@@ +""" + +_SNAP4_IMAGE_RE = re.compile(r"doca_snap[^\s:@\"']*:v?(\d+\.\d+\.\d+)", re.IGNORECASE) + + +def snap4_version_from_image(image: str) -> str | None: + match = _SNAP4_IMAGE_RE.search(image or "") + return match.group(1) if match else None + + +def _snap4_sections(text: str) -> dict[str, str]: + sections: dict[str, list[str]] = {} + current: str | None = None + for line in (text or "").splitlines(): + marker = line.strip() + if marker.startswith("@@SNAP4-") and marker.endswith("@@"): + current = marker[len("@@SNAP4-") : -2] + sections.setdefault(current, []) + continue + if current is not None: + sections[current].append(line) + return {name: "\n".join(lines) for name, lines in sections.items()} + + +def _json_or_none(text: str) -> Any: + try: + return json.loads(text) + except (json.JSONDecodeError, TypeError): + return None + + +def parse_snap4_listing(text: str) -> dict[str, Any]: + """Read the SNAP4_DPU_SCRIPT output. + + ``reached`` says the script ran on the DPU at all. ``listed`` says crictl + answered, which is what makes "no SNAP-4 container here" a finding rather + than a guess. ``running`` and ``configured`` are the versions read off the + running containers and the kubelet manifests. + """ + sections = _snap4_sections(text) + result: dict[str, Any] = { + "reached": "MANIFESTS" in sections, + "listed": False, + "running": [], + "runningWithoutVersion": False, + "configured": [], + "error": None, + } + if not result["reached"]: + result["error"] = "the DPU-side read produced no output" + return result + for line in sections.get("MANIFESTS", "").splitlines(): + version = snap4_version_from_image(line) + if version and version not in result["configured"]: + result["configured"].append(version) + if "NO-CRICTL" in sections: + result["error"] = "crictl is not installed on the DPU" + return result + ps_text = sections.get("PS", "") + if "PS-FAILED" in sections: + result["error"] = "crictl ps did not answer, even under sudo -n" + return result + containers = (_json_or_none(ps_text) or {}).get("containers") + if not isinstance(containers, list): + result["error"] = "crictl ps returned no readable container list" + return result + result["listed"] = True + tags: dict[str, list[str]] = {} + images = (_json_or_none(sections.get("IMAGES", "")) or {}).get("images") or [] + for image in images if isinstance(images, list) else []: + if not isinstance(image, dict): + continue + names = [str(tag) for tag in image.get("repoTags") or []] + for key in (image.get("id"), *(image.get("repoDigests") or [])): + if key: + tags[str(key)] = names + for container in containers: + if not isinstance(container, dict): + continue + image_field = container.get("image") + image_name = str(image_field.get("image") if isinstance(image_field, dict) else image_field or "") + names = [image_name, *tags.get(image_name, []), *tags.get(str(container.get("imageRef") or ""), [])] + container_name = str((container.get("metadata") or {}).get("name") or "") + is_snap = any("doca_snap" in name.lower() for name in names) or "snap" in container_name.lower() + if not is_snap: + continue + versions = [version for version in map(snap4_version_from_image, names) if version] + if versions: + if versions[0] not in result["running"]: + result["running"].append(versions[0]) + else: + result["runningWithoutVersion"] = True + return result + + +def collect_snap4( + rshim: dict[str, Any], + *, + on_dpu: bool, + env: dict[str, str] = os.environ, + runner: Runner = subprocess.run, +) -> dict[str, Any]: + """Read the SNAP-4 container version on the DPU, if this host can reach it.""" + if on_dpu: + command, via, timeout = ["sh", "-c", SNAP4_DPU_SCRIPT], "local", TOOL_TIMEOUT_S + elif env.get("CLUSTERMAX_DPU_SSH_DISABLE"): + return {"attempted": False, "via": None, "reason": "RShim SSH is disabled"} + elif not (rshim.get("rshimDeviceNode") or rshim.get("tmfifoNet0")): + return {"attempted": False, "via": None, "reason": "no RShim link to the DPU"} + else: + command, via, timeout = [*dpu_ssh_prefix(env), SNAP4_DPU_SCRIPT], "rshim-ssh", SSH_TIMEOUT_S * 2 + text, error = run_tool(command, runner=runner, timeout=timeout) + listing = parse_snap4_listing(text) + listing["attempted"] = True + listing["via"] = via + if not listing["reached"]: + listing["error"] = error or listing["error"] + elif via == "rshim-ssh": + # The script ran on the DPU, so the host did reach it. + rshim["dpuReachedFromHost"] = True + return listing + + +def decide_snap4(record: dict[str, Any], read: dict[str, Any]) -> dict[str, Any]: + """Per-host SNAP-4 state, from the host record and the DPU-side read.""" + via = read.get("via") + if read.get("running"): + return { + "state": STATE_VERSION, + "versions": list(read["running"]), + "source": f"{via}/crictl", + "reason": "SNAP-4 image tag read from the running container", + } + if read.get("configured"): + return { + "state": STATE_VERSION, + "versions": list(read["configured"]), + "source": f"{via}/kubelet-manifest", + "reason": ( + "SNAP-4 image tag read from the kubelet manifest; no running " + "container was listed, and kubelet starts this version on restart" + ), + } + base = {"versions": [], "source": None} + if read.get("runningWithoutVersion"): + return { + **base, + "state": STATE_UNKNOWN, + "reason": "a SNAP-4 container runs on the DPU but its image reference carries no version tag", + } + if not record.get("scanComplete"): + return {**base, "state": STATE_INCOMPLETE, "reason": str(record.get("scanError") or "PCI scan could not run")} + if not record.get("bluefield3Present"): + return {**base, "state": STATE_NOT_APPLICABLE, "reason": "no BlueField-3 device is present on this host"} + if record.get("mode") == "nic": + return { + **base, + "state": STATE_NOT_RUNNING, + "reason": ( + "BlueField-3 is in NIC mode (INTERNAL_CPU_OFFLOAD_ENGINE=1), so the DPU " + "Arm cores that run SNAP-4 are off. A change back to DPU mode would " + "start any installed SNAP-4 container unchanged." + ), + } + if read.get("listed"): + return { + **base, + "state": SNAP4_STATE_NOT_INSTALLED, + "source": f"{via}/crictl", + "reason": "the DPU answered and runs no SNAP-4 container, and no kubelet manifest names one", + } + if record.get("mode") != "dpu": + return { + **base, + "state": STATE_INCOMPLETE, + "reason": "BlueField-3 is present but its mode could not be read, so SNAP-4 scope is unsettled", + } + if read.get("reached"): + detail = f"the DPU answered but {read.get('error') or 'listed nothing'}" + elif read.get("attempted"): + detail = f"the DPU did not answer over RShim ({read.get('error') or 'no output'})" + else: + detail = str(read.get("reason") or "the DPU was not reached") + return { + **base, + "state": STATE_UNKNOWN, + "reason": ( + "BlueField-3 is in DPU mode and the SNAP-4 container is observable only " + f"on the DPU Arm side; {detail}. The provider must attest the SNAP-4 " + "version or confirm it is not deployed." + ), + } + + +# Most severe first. not_running ranks above version for the same reason as on +# the controller ladder: a NIC-mode host was not read, so a read peer must not +# speak for it in the reported state. The evaluator grades the readings +# themselves from `observed`, so this order never softens a fail. +SNAP4_LADDER = ( + STATE_UNKNOWN, + STATE_INCOMPLETE, + STATE_NOT_RUNNING, + STATE_VERSION, + SNAP4_STATE_NOT_INSTALLED, + STATE_NOT_APPLICABLE, +) +SNAP4_GAP_RUNG = STATE_INCOMPLETE + + +def _snap4_rung(record: dict[str, Any]) -> str: + snap = record.get("snap4") if isinstance(record.get("snap4"), dict) else {} + state = str(snap.get("state") or STATE_INCOMPLETE) + return state if state in SNAP4_LADDER else STATE_INCOMPLETE + + +def summarize_snap4(records: list[dict[str, Any]]) -> dict[str, Any]: + """Roll the per-host SNAP-4 states into the cluster answer. + + ``observed`` carries every distinct version a host read, whatever the + rollup state, so a below-minimum DPU stays a finding beside an unread peer. + SNAP-4 is fixed per release train, so the readings are not ordered here; the + evaluator grades each one against its own train. + """ + if not records: + return { + "state": STATE_INCOMPLETE, + "reason": "no host produced a SNAP-4 record", + "observed": [], + "hostsChecked": 0, + } + state = _snap4_rung(most_severe(records, ladder=SNAP4_LADDER, rung_of=_snap4_rung)) + seen: dict[tuple[str, str], dict[str, Any]] = {} + for record in records: + snap = record.get("snap4") if isinstance(record.get("snap4"), dict) else {} + for version in snap.get("versions") or []: + key = (str(version), str(snap.get("source") or "")) + seen.setdefault(key, {"version": str(version), "host": record.get("host"), "source": snap.get("source")}) + observed = sorted(seen.values(), key=lambda entry: (_version_key(entry["version"]), str(entry["host"] or ""))) + reasons = sorted( + { + str((record.get("snap4") or {}).get("reason") or "") + for record in records + if _snap4_rung(record) == state + } + ) + return { + "state": state, + "reason": "; ".join(reason for reason in reasons if reason), + "observed": observed, + "hostsChecked": len(records), + } + + # --------------------------------------------------------------------------- # State machine # --------------------------------------------------------------------------- @@ -985,6 +1284,13 @@ def gather( "lineEvidence": version.get("lineEvidence") or [], } record["state"], record["reason"] = decide(record) + on_dpu = bool(version.get("versionSource")) and not str(version["versionSource"]).startswith("rshim-ssh/") + if record["bluefield3Present"] and record["mode"] != "nic": + snap_read = collect_snap4(rshim, on_dpu=on_dpu, env=env, runner=runner) + else: + # No DPU, or its Arm cores are off: there is nothing to reach. + snap_read = {"attempted": False, "reason": "no DPU Arm side to read"} + record["snap4"] = decide_snap4(record, snap_read) return record @@ -1117,6 +1423,7 @@ def summarize(records: list[dict[str, Any]]) -> dict[str, Any]: "bluefield3Present": None, "rshimHostAccess": {}, }, + "snap4": summarize_snap4(records), } observed = observed_controllers(records) @@ -1170,6 +1477,7 @@ def summarize(records: list[dict[str, Any]]) -> dict[str, Any]: "observedControllers": observed, "worstObserved": worst_observed(observed), "isolationEvidence": isolation_evidence(records), + "snap4": summarize_snap4(records), } @@ -1229,6 +1537,16 @@ def apply_coverage_gap(summary: dict[str, Any], errors: list[str]) -> dict[str, evidence["scanComplete"] = False evidence["scanError"] = f"the per-node scan did not reach every host ({gap})" summary["isolationEvidence"] = evidence + + # SNAP-4 is per DPU too, so the same gap withdraws its clean claims. The + # readings in `observed` stay, so a below-minimum DPU is still a finding. + snap4 = dict(summary.get("snap4") or {}) + if snap4 and gap_can_weaken(snap4.get("state"), ladder=SNAP4_LADDER, gap_rung=SNAP4_GAP_RUNG): + snap4["state"] = SNAP4_GAP_RUNG + prior_snap = str(snap4.get("reason") or "").strip().rstrip(";") + snap_note = f"the per-node scan did not reach every host ({gap})" + snap4["reason"] = f"{prior_snap}; {snap_note}" if prior_snap else snap_note + summary["snap4"] = snap4 return summary @@ -1572,6 +1890,11 @@ def record_from_k8s(node: str, parsed: dict[str, Any]) -> dict[str, Any]: "lineEvidence": [parsed["virtnet"]] if version else [], } record["state"], record["reason"] = decide(record) + # The driver pod reads the host, not the DPU, so SNAP-4 stays unread here. + record["snap4"] = decide_snap4( + record, + {"attempted": False, "reason": "the Kubernetes driver-pod read does not reach the DPU"}, + ) return record @@ -1782,6 +2105,7 @@ def collector_summary(payload: dict[str, Any]) -> dict[str, Any]: "virtioNetWorstObservedHost": worst.get("host") or "", "virtioNetObservedJson": json.dumps(observed, sort_keys=True), "dpuIsolationJson": json.dumps(summary.get("isolationEvidence") or {}, sort_keys=True), + "snap4Json": json.dumps(summary.get("snap4") or {}, sort_keys=True), "state": state, "reason": summary.get("reason") or "", } diff --git a/cmax/scripts/1-audit/security_fleet.py b/cmax/scripts/1-audit/security_fleet.py index 8a4d8e1..991f386 100644 --- a/cmax/scripts/1-audit/security_fleet.py +++ b/cmax/scripts/1-audit/security_fleet.py @@ -29,9 +29,9 @@ from typing import Any # Checks whose evidence lives on each GPU host. Cluster-level checks (the -# dcgm-exporter image, the CUDA runtime pods, and the BlueField VIRTIO-Net and -# DPU host-isolation results, which both come from the fabric-wide virtio-net -# check) already have their own scope and are left as the collector built them. +# dcgm-exporter image, the CUDA runtime pods, and the BlueField VIRTIO-Net, +# SNAP-4, and DPU host-isolation results, which all come from the fabric-wide +# virtio-net check) already have their own scope and are left as the collector built them. VERSION_KEYS = ( "nvidiaDriver", "amdDriver", diff --git a/cmax/scripts/1-audit/security_version_audit.py b/cmax/scripts/1-audit/security_version_audit.py index f08b05b..ef93252 100644 --- a/cmax/scripts/1-audit/security_version_audit.py +++ b/cmax/scripts/1-audit/security_version_audit.py @@ -437,6 +437,195 @@ def dcgm_exporter_verdict(version: str) -> Verdict: return _minimum_kind_verdict(version, "dcgmExporter", label="DCGM Exporter") +# --------------------------------------------------------------------------- +# SNAP-4 container (NVIDIA bulletin a_id 5744) +# --------------------------------------------------------------------------- + +SNAP4_COMPONENT = "snap4Container" +# Rollup states where every host was either read or cannot run SNAP-4 now, so a +# set of passing readings speaks for the cluster. +SNAP4_COMPLETE_STATES = frozenset({"version", "not_installed", "not_running", "not_applicable"}) + + +def _snap4_trains() -> tuple[dict[str, str], str, dict[str, Any]]: + """(train -> fixed version, minimum text, component block). Raises on a missing table.""" + block = minimum_versions.component(SNAP4_COMPONENT) + lines = block.get("lines") if isinstance(block.get("lines"), dict) else {} + fixed = { + str(train): str(entry["fixed"]) + for train, entry in lines.items() + if isinstance(entry, dict) and numeric_version(str(entry.get("fixed") or "")) is not None + } + ordered = sorted(fixed.items(), key=lambda item: numeric_version(item[1]) or ()) + text = ", ".join(f"{version} ({train} train)" for train, version in ordered) + return fixed, text or MINIMUMS_UNAVAILABLE, block + + +def _snap4_minimum_text() -> str: + try: + return _snap4_trains()[1] + except MinimumDataError: + return MINIMUMS_UNAVAILABLE + + +def snap4_verdict(version: str) -> tuple[Verdict, str | None]: + """Grade one SNAP-4 version against the fixed version of its own train. + + NVIDIA fixes SNAP-4 per major.minor train ("SNAP-4.9.1, SNAP4 4.5.5"), and + the bulletin marks everything below the newest fix as affected except the + fixed trains themselves. So a version on a listed train is held to that + train's fix, a version newer than every fix carries them all, and a version + between or below the trains has no fix of its own and must move to the next + fixed release above it. Returns the verdict and the train its minimum came + from, which selects the grace window. + """ + try: + fixed, minimum_text, block = _snap4_trains() + except MinimumDataError as exc: + return _minimums_unavailable(version, exc), None + advisory = _advisory(block) + cited = _cve_detail(block, "SNAP-4 security bulletin minimum") + if not fixed: + return ( + _unknown(version, MINIMUMS_UNAVAILABLE, advisory, "snap4Container lists no fixed release train"), + None, + ) + observed = numeric_version(version, parts=3) + if observed is None: + return _unknown(version, minimum_text, advisory, f"SNAP-4 version {version!r} is unreadable"), None + train = f"{observed[0]}.{observed[1]}" + if train in fixed: + minimum = fixed[train] + passed = observed >= (numeric_version(minimum, parts=3) or ()) + detail = ( + f"Meets the {cited} for the SNAP-4 {train} train ({minimum})" + if passed + else f"Below the {cited} for the SNAP-4 {train} train ({minimum})" + ) + return Verdict(version, "pass" if passed else "fail", minimum, advisory, detail), train + ordered = sorted(fixed.items(), key=lambda item: numeric_version(item[1], parts=3) or ()) + highest_train, highest = ordered[-1] + if observed > (numeric_version(highest, parts=3) or ()): + return ( + Verdict( + version, + "pass", + highest, + advisory, + f"Newer than every fixed SNAP-4 train in the {cited} (newest fix {highest}), " + "so it carries those fixes", + ), + highest_train, + ) + above = next( + (item for item in ordered if (numeric_version(item[1], parts=3) or ()) > observed), + ordered[-1], + ) + return ( + Verdict( + version, + "fail", + above[1], + advisory, + f"Below the {cited}: the SNAP-4 {train} train has no fixed release, so the " + f"nearest fix is {above[1]} ({above[0]} train)", + ), + above[0], + ) + + +def snap4_container_result(summary: dict[str, Any] | None) -> dict[str, object]: + """Grade the SNAP-4 rollup the virtio-net check produced. + + Every reading is graded on its own train and the worst one decides. A + below-minimum reading fails whatever the coverage, and a passing reading + clears the cluster only when every host was read or cannot run SNAP-4 now. + """ + advisory = _component_advisory(SNAP4_COMPONENT) + minimum_text = _snap4_minimum_text() + if not isinstance(summary, dict) or not summary: + return asdict( + _unknown( + "unknown", + minimum_text, + advisory, + "The SNAP-4 container read did not run, so its version was not observed", + ) + ) + state = str(summary.get("state") or "incomplete") + reason = str(summary.get("reason") or "").strip() + observed = [entry for entry in summary.get("observed") or [] if isinstance(entry, dict) and entry.get("version")] + complete = state in SNAP4_COMPLETE_STATES + + def finish(verdict: Verdict, selector: str | None, entry: dict[str, Any] | None) -> dict[str, object]: + record = _verdict_record(verdict, SNAP4_COMPONENT if selector else None, selector) + record["state"] = state + if entry: + record["host"] = entry.get("host") + record["source"] = entry.get("source") + if observed: + record["observed"] = observed + return record + + graded = [(entry, *snap4_verdict(str(entry["version"]))) for entry in observed] + for status in ("fail", "unknown"): + hits = [item for item in graded if item[1].status == status] + if not hits: + continue + entry, verdict, selector = min(hits, key=lambda item: numeric_version(item[1].version, parts=3) or ()) + detail = f"{verdict.detail}, read on {entry.get('host') or 'a host'} via {entry.get('source') or 'unknown'}" + if not complete and status == "fail": + detail += f". Not every DPU was read ({reason or state})" + return finish(replace(verdict, detail=detail), selector if status == "fail" else None, entry) + if graded: + entry, verdict, selector = min(graded, key=lambda item: numeric_version(item[1].version, parts=3) or ()) + if complete: + return finish(verdict, selector, entry) + return finish( + _unknown( + verdict.version, + verdict.minimum, + verdict.advisory, + f"Every SNAP-4 version read meets its train's fix, but not every DPU was read: {reason or state}", + ), + None, + entry, + ) + if state == "not_applicable": + verdict = Verdict( + "not-installed", + "not_applicable", + minimum_text, + advisory, + "No BlueField-3 DPU is present in the completed scan, so SNAP-4 does not apply", + ) + elif state == "not_installed": + verdict = Verdict( + "not-installed", + "not_applicable", + minimum_text, + advisory, + "Every reached BlueField-3 DPU runs no SNAP-4 container and no kubelet manifest names one", + ) + elif state == "not_running": + verdict = Verdict( + "not-running", + "not_applicable", + minimum_text, + advisory, + "BlueField-3 is in NIC mode, so the DPU Arm cores that run SNAP-4 are off. " + "Recheck SNAP-4 before switching back to DPU mode.", + ) + else: + verdict = _unknown( + "unknown", + minimum_text, + advisory, + reason or "No DPU reported a SNAP-4 container version", + ) + return finish(verdict, None, None) + + def _meets_ladder_minimum(version: str, observed: tuple[int, ...], minimum_version: str) -> bool: """Compare one observed release against its branch minimum in the ladder. @@ -1739,6 +1928,7 @@ def evaluate( nvidia_gpu_present: bool | None = None, amd_model: str = "unknown", amd_driver_evidence: dict[str, Any] | None = None, + snap4: dict[str, Any] | None = None, ) -> dict[str, object]: if gpu_vendor.lower() == "nvidia": driver_result = nvidia_driver_verdict(driver) @@ -1848,6 +2038,7 @@ def evaluate( "dcgmExporter": _verdict_record( dcgm_exporter_result, "dcgmExporter" ), + "snap4Container": snap4_container_result(snap4), "floorsMetadata": minimums_metadata(), } @@ -1942,6 +2133,11 @@ def main() -> int: help="virtio-net check record JSON carrying scanComplete, bluefield3Present, " "and rshimHostAccess", ) + parser.add_argument( + "--snap4-json", + default=None, + help="the virtio-net check's SNAP-4 rollup (state, reason, observed)", + ) args = parser.parse_args() print( json.dumps( @@ -1971,6 +2167,7 @@ def main() -> int: nvidia_gpu_present=False if args.nvidia_gpu_absent else None, dcgm=args.dcgm, dcgm_exporter=args.dcgm_exporter, + snap4=_parse_isolation_json(args.snap4_json), ), sort_keys=True, ) diff --git a/cmax/security.py b/cmax/security.py index 969d42d..5c72fee 100644 --- a/cmax/security.py +++ b/cmax/security.py @@ -1549,6 +1549,57 @@ def _virtio_net_bluefield(audit: dict[str, Any]) -> tuple[str, str, str]: ) +def _snap4_container(audit: dict[str, Any]) -> tuple[str, str, str]: + """Grade the BlueField SNAP-4 container against its release-train minimum. + + SNAP-4 runs on the DPU Arm cores, so the audit reads it on the same DPU + path as the virtio-net controller. The audit already graded every reading + against its own train; this only words the result. + """ + verdict = _get(audit, "securityVersions.snap4Container") + if not isinstance(verdict, dict): + return ( + WARNING, + "not collected", + "The BlueField SNAP-4 read was unavailable, so exposure cannot be ruled out.", + ) + status = str(verdict.get("status") or "unknown").lower() + version = _display(verdict.get("version")) + minimum = _display(verdict.get("minimum")) + detail = str(verdict.get("detail") or "").strip() + if status in {"not_applicable", "not-applicable"}: + state = str(verdict.get("state") or "") + observed = { + "not_installed": "no SNAP-4 container on any reached DPU", + "not_running": "no SNAP-4 running; card in NIC mode", + }.get(state, "no BlueField DPU present") + return NOT_APPLICABLE, observed, detail or "SNAP-4 does not apply to this cluster." + if status == "pass": + grace_active, grace_note = _grace_details(verdict) + if grace_active: + return ( + PASS, + f"observed version {version}; minimum version {minimum} during vendor update window", + detail or grace_note, + ) + return ( + PASS, + f"observed version {version}; minimum version {minimum}", + "The SNAP-4 container is at or above the fix for its release train.", + ) + if status == "fail": + return ( + CRITICAL, + f"observed version {version}; minimum version {minimum}", + detail or "The SNAP-4 container is below the fix for its release train.", + ) + return ( + WARNING, + f"observed version {version}; minimum version {minimum}", + detail or "The BlueField SNAP-4 container version requires provider attestation.", + ) + + def _dpu_host_isolation(audit: dict[str, Any]) -> tuple[str, str, str]: """Grade whether the host can reach the DPU control plane over RShim. @@ -2257,6 +2308,24 @@ def _observability(audit: dict[str, Any]) -> tuple[str, str, str]: ), _dpu_host_isolation, ), + CheckSpec( + "snap4-container", + "BlueField SNAP-4 container", + "A SNAP-4 container below the fix for its release train exposes the " + "DPU storage emulation service to the flaws in NVIDIA's SNAP-4 bulletin.", + "Update the SNAP-4 container to the fixed release for its train, or to " + "a newer train. A card in NIC mode is not running SNAP-4 now, and a " + "switch to DPU mode starts it.", + # The bulletin link and CVEs come from the minimum table (see + # MINIMUM_BOUND_CHECKS). + _references( + ( + "NVIDIA product security", + "https://www.nvidia.com/en-us/security/", + ), + ), + _snap4_container, + ), ) @@ -2313,6 +2382,14 @@ def _observability(audit: dict[str, Any]) -> tuple[str, str, str]: "switch to DPU mode activates it.", "NVIDIA BlueField VIRTIO-Net bulletin", ), + "snap4-container": ( + "snap4Container", + # Per release train, so no single minimum to name. + "Update the SNAP-4 container to the fixed release for its train, or to " + "a newer train. A card in NIC mode is not running SNAP-4 now, and a " + "switch to DPU mode starts it.", + "NVIDIA Networking SNAP4 bulletin", + ), } @@ -2519,6 +2596,7 @@ def evaluate_security(values: dict[str, Any]) -> list[SecurityCheck]: "docker": "securityVersions.docker", "connectx-firmware": "securityVersions.connectxFirmware", "virtio-net-bluefield": "securityVersions.virtioNetBluefield", + "snap4-container": "securityVersions.snap4Container", "fragnesia": "security.fragnesia", "guest-kernel": "security.guestKernel.ubuntuCveGracePeriods", } diff --git a/tests/audit/test_security_cli.py b/tests/audit/test_security_cli.py index d3e7f0f..ed2a33b 100644 --- a/tests/audit/test_security_cli.py +++ b/tests/audit/test_security_cli.py @@ -40,6 +40,7 @@ "docker": "https://example.invalid/minimums/docker", "connectxFirmware": "https://example.invalid/minimums/connectx", "virtioNetBluefield": "https://example.invalid/minimums/virtio", + "snap4Container": "https://example.invalid/minimums/snap4", } @@ -91,6 +92,13 @@ def minimum_table(generated: datetime, **components: dict) -> dict: "cves": ["CVE-2026-65094"], "source": {"aId": 5815, "feed": "nvidia-csaf"}, }, + "snap4Container": { + "kind": "releaseLines", + "lines": {"4.5": {"fixed": "4.5.5"}, "4.9": {"fixed": "4.9.1"}}, + "advisory": FIXTURE_ADVISORIES["snap4Container"], + "cves": ["CVE-2025-33215", "CVE-2025-33216"], + "source": {"aId": 5744, "feed": "nvidia-csaf"}, + }, }, } @@ -866,7 +874,7 @@ def test_default_report_shows_every_check_and_observed_versions(self) -> None: Path("/tmp/report"), ) self.assertIn("# ClusterMAX security audit report", report) - self.assertIn("3 failed, 12 warnings, 4 passed, 3 skipped", report) + self.assertIn("3 failed, 13 warnings, 4 passed, 3 skipped", report) self.assertIn("Action required: 3 critical findings.", report) self.assertNotIn("critical findings are known exposure", report) self.assertIn("NVIDIA driver minimum version", report) @@ -1407,12 +1415,12 @@ def test_staleness_changes_no_count_and_no_exit_code(self) -> None: security.counts(stale_checks), { security.PASS: 3, - security.WARNING: 12, + security.WARNING: 13, security.CRITICAL: 4, security.NOT_APPLICABLE: 3, }, ) - self.assertEqual(sum(security.counts(stale_checks).values()), 22) + self.assertEqual(sum(security.counts(stale_checks).values()), 23) self.assertEqual(clean_exit, 0) report = self._report(now=self.generated + timedelta(days=40), verbosity=3) self.assertNotIn("minimum-version-freshness", report) @@ -1467,8 +1475,8 @@ def test_saved_report_records_the_minimum_state(self) -> None: def test_minimum_freshness_is_not_a_graded_criterion(self) -> None: ids = [spec.id for spec in security.CHECK_SPECS] - self.assertEqual(len(ids), 22) - self.assertEqual(len(set(ids)), 22) + self.assertEqual(len(ids), 23) + self.assertEqual(len(set(ids)), 23) self.assertEqual( ids, [ @@ -1494,6 +1502,7 @@ def test_minimum_freshness_is_not_a_graded_criterion(self) -> None: "nvlink-boundary", "virtio-net-bluefield", "dpu-host-isolation", + "snap4-container", ], ) diff --git a/tests/audit/test_snap4_container.py b/tests/audit/test_snap4_container.py new file mode 100644 index 0000000..65ca471 --- /dev/null +++ b/tests/audit/test_snap4_container.py @@ -0,0 +1,389 @@ +"""The BlueField SNAP-4 container read and its per-train grade. + +SNAP-4 runs on the DPU Arm cores, so the virtio-net check reads it on the same +DPU path. These tests pin the per-host states, the cluster rollup, and the +security verdict, including the per-train rule from NVIDIA bulletin 5744. + +Sources: cmax/scripts/1-audit/checks/fabric/virtio-net-check.py (the read and +rollup) and cmax/scripts/1-audit/security_version_audit.py (the grade). +""" + +from __future__ import annotations + +import json +import os +import shutil +import unittest +from pathlib import Path + +from tests.audit.test_security_version_audit import security +from tests.audit.test_virtio_net_check import ( + BLUEFIELD3_LSPCI, + DPU_MODE_MLXCONFIG, + NIC_MODE_MLXCONFIG, + NO_BLUEFIELD_LSPCI, + VIRTNET_VERSION_JSON, + CheckHarness, + load_check_module, +) + +minimums = security.minimum_versions + +SNAP_IMAGE_ID = "sha256:5d3f0b2c9a" +RUNNING_PS = json.dumps( + { + "containers": [ + { + "id": "c1", + "metadata": {"name": "doca-snap"}, + "image": {"image": SNAP_IMAGE_ID}, + "imageRef": SNAP_IMAGE_ID, + "state": "CONTAINER_RUNNING", + }, + { + "id": "c2", + "metadata": {"name": "doca-telemetry"}, + "image": {"image": "sha256:other"}, + "imageRef": "sha256:other", + }, + ] + }, + indent=2, +) +IMAGES = json.dumps( + { + "images": [ + {"id": SNAP_IMAGE_ID, "repoTags": ["nvcr.io/nvidia/doca/doca_snap:4.9.0-doca3.1.0"]}, + {"id": "sha256:other", "repoTags": ["nvcr.io/nvidia/doca/doca_telemetry:1.21.4-doca3.1.0"]}, + ] + }, + indent=2, +) + + +def listing(*, manifests: str = "", ps: str | None = RUNNING_PS, images: str = IMAGES) -> str: + """What SNAP4_DPU_SCRIPT prints on a DPU.""" + lines = ["@@SNAP4-MANIFESTS@@", manifests] + if ps is None: + lines += ["@@SNAP4-NO-CRICTL@@"] + else: + lines += ["@@SNAP4-PS@@", ps, "@@SNAP4-IMAGES@@", images, "@@SNAP4-END@@"] + return "\n".join(lines) + "\n" + + +class Snap4ListingParseTests(unittest.TestCase): + def setUp(self) -> None: + self.check = load_check_module() + + def test_running_container_version_comes_from_the_image_tag(self) -> None: + parsed = self.check.parse_snap4_listing(listing()) + self.assertTrue(parsed["reached"]) + self.assertTrue(parsed["listed"]) + self.assertEqual(parsed["running"], ["4.9.0"]) + + def test_container_named_by_its_image_needs_no_images_listing(self) -> None: + ps = json.dumps( + {"containers": [{"metadata": {"name": "x"}, "image": {"image": "nvcr.io/nvidia/doca/doca_snap:4.5.5-doca2.9.1"}}]} + ) + parsed = self.check.parse_snap4_listing(listing(ps=ps, images="{}")) + self.assertEqual(parsed["running"], ["4.5.5"]) + + def test_manifest_version_is_read_when_nothing_runs(self) -> None: + parsed = self.check.parse_snap4_listing( + listing( + manifests=" image: nvcr.io/nvidia/doca/doca_snap:4.9.1-doca3.2.0", + ps=json.dumps({"containers": []}), + ) + ) + self.assertEqual(parsed["running"], []) + self.assertEqual(parsed["configured"], ["4.9.1"]) + + def test_a_snap_container_without_a_tag_is_flagged(self) -> None: + ps = json.dumps({"containers": [{"metadata": {"name": "snap"}, "image": {"image": "sha256:none"}}]}) + parsed = self.check.parse_snap4_listing(listing(ps=ps, images="{}")) + self.assertEqual(parsed["running"], []) + self.assertTrue(parsed["runningWithoutVersion"]) + + def test_no_crictl_and_no_output_are_not_a_listing(self) -> None: + self.assertFalse(self.check.parse_snap4_listing(listing(ps=None))["listed"]) + empty = self.check.parse_snap4_listing("") + self.assertFalse(empty["reached"]) + self.assertFalse(empty["listed"]) + + def test_failed_ps_is_not_a_listing(self) -> None: + text = "@@SNAP4-MANIFESTS@@\n@@SNAP4-PS@@\n@@SNAP4-PS-FAILED@@\n" + parsed = self.check.parse_snap4_listing(text) + self.assertTrue(parsed["reached"]) + self.assertFalse(parsed["listed"]) + + +class Snap4HostReadTests(CheckHarness): + def bluefield(self, mode: str = DPU_MODE_MLXCONFIG) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + self.stub_output("mlxconfig", mode) + (self.root / "dev" / "rshim0").touch() + + def ssh_prints(self, text: str, *, returncode: int = 0) -> None: + data = self.tmp / "ssh-snap4.stdout" + data.write_text(text) + # virtnet over SSH fails; the SNAP-4 script (the argument carrying + # crictl) prints the canned DPU listing. + self.stub( + "ssh", + f""" +for arg in "$@"; do + case "$arg" in + *crictl*) /bin/cat "{data}"; exit {returncode} ;; + esac +done +exit 1 +""", + ) + + def snap_calls(self) -> str: + """The SSH call that carried the SNAP-4 script, or "" if none did.""" + path = self.calls / "ssh.calls" + text = path.read_text() if path.exists() else "" + return text if "crictl" in text else "" + + def test_running_container_is_read_over_rshim(self) -> None: + self.bluefield() + self.ssh_prints(listing()) + + record = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="") + + snap = record["snap4"] + self.assertEqual(snap["state"], "version") + self.assertEqual(snap["versions"], ["4.9.0"]) + self.assertEqual(snap["source"], "rshim-ssh/crictl") + self.assertTrue(record["rshimHostAccess"]["dpuReachedFromHost"]) + argv = self.snap_calls() + self.assertIn("BatchMode=yes", argv) + self.assertIn("sudo -n crictl", argv) + # The read lists only. It never pulls, runs, stops, or execs. + for verb in ("crictl pull", "crictl run", "crictl stop", "crictl exec", "crictl rm"): + self.assertNotIn(verb, argv) + + def test_collector_summary_carries_the_rollup(self) -> None: + self.bluefield() + self.ssh_prints(listing()) + + summary = self.run_check("--summary", CLUSTERMAX_DPU_SSH_DISABLE="") + + snap = json.loads(summary["snap4Json"]) + self.assertEqual(snap["state"], "version") + self.assertEqual(snap["observed"][0]["version"], "4.9.0") + + def test_reached_dpu_without_snap_is_not_installed(self) -> None: + self.bluefield() + self.ssh_prints(listing(ps=json.dumps({"containers": []}))) + + snap = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="")["snap4"] + + self.assertEqual(snap["state"], "not_installed") + self.assertEqual(snap["versions"], []) + + def test_unreachable_dpu_is_unknown_and_asks_for_attestation(self) -> None: + self.bluefield() + self.stub("ssh", 'echo "Permission denied (publickey)." >&2\nexit 255') + + snap = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="")["snap4"] + + self.assertEqual(snap["state"], "unknown") + self.assertIn("attest", snap["reason"]) + + def test_disabled_ssh_is_unknown_without_a_call(self) -> None: + self.bluefield() + self.ssh_prints(listing()) + + snap = self.host_record()["snap4"] + + self.assertEqual(snap["state"], "unknown") + self.assertEqual(self.snap_calls(), "") + + def test_nic_mode_is_not_running_and_never_dials_the_dpu(self) -> None: + self.bluefield(NIC_MODE_MLXCONFIG) + self.ssh_prints(listing()) + + snap = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="")["snap4"] + + self.assertEqual(snap["state"], "not_running") + self.assertEqual(self.snap_calls(), "") + + def test_no_bluefield_is_not_applicable(self) -> None: + self.stub_output("lspci", NO_BLUEFIELD_LSPCI) + self.ssh_prints(listing()) + + snap = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="")["snap4"] + + self.assertEqual(snap["state"], "not_applicable") + self.assertEqual(self.snap_calls(), "") + + def test_on_the_dpu_the_read_runs_locally(self) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + self.stub_output("mlxconfig", DPU_MODE_MLXCONFIG) + payload = self.tmp / "virtnet-version.json" + payload.write_text(VIRTNET_VERSION_JSON) + self.stub("virtnet", f'case "$1" in version) /bin/cat "{payload}" ;; *) exit 1 ;; esac') + for tool in ("sh", "grep"): + os.symlink(shutil.which(tool), self.bin / tool) + (self.tmp / "ps.json").write_text(RUNNING_PS) + (self.tmp / "images.json").write_text(IMAGES) + self.stub( + "crictl", + f'case "$1" in ps) /bin/cat "{self.tmp}/ps.json" ;; images) /bin/cat "{self.tmp}/images.json" ;; esac', + ) + self.ssh_prints(listing()) + + snap = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="")["snap4"] + + self.assertEqual(snap["versions"], ["4.9.0"]) + self.assertEqual(snap["source"], "local/crictl") + self.assertEqual(self.snap_calls(), "") + + +class Snap4SummaryTests(unittest.TestCase): + def setUp(self) -> None: + self.check = load_check_module() + + def record(self, host: str, state: str, versions=(), source="rshim-ssh/crictl", reason="r") -> dict: + return { + "host": host, + "state": "version", + "snap4": {"state": state, "versions": list(versions), "source": source, "reason": reason}, + } + + def test_every_reading_survives_the_rollup(self) -> None: + summary = self.check.summarize_snap4( + [ + self.record("a", "version", ["4.9.1"]), + self.record("b", "version", ["4.7.0"]), + self.record("c", "unknown", reason="DPU not reached"), + ] + ) + self.assertEqual(summary["state"], "unknown") + self.assertEqual([entry["version"] for entry in summary["observed"]], ["4.7.0", "4.9.1"]) + self.assertEqual(summary["hostsChecked"], 3) + + def test_nic_peer_outranks_a_read_host(self) -> None: + summary = self.check.summarize_snap4( + [self.record("a", "version", ["4.9.1"]), self.record("b", "not_running")] + ) + self.assertEqual(summary["state"], "not_running") + + def test_a_coverage_gap_withdraws_a_clean_rollup(self) -> None: + summary = {"snap4": {"state": "not_installed", "reason": "", "observed": []}} + gapped = self.check.apply_coverage_gap(dict(summary), ["node-7: srun failed"]) + self.assertEqual(gapped["snap4"]["state"], "incomplete") + + +SNAP4_BLOCK = { + "kind": "releaseLines", + "lines": {"4.5": {"fixed": "4.5.5"}, "4.9": {"fixed": "4.9.1"}}, + "advisory": "https://nvidia.custhelp.com/app/answers/detail/a_id/5744", + "cves": ["CVE-2025-33215", "CVE-2025-33216"], + "source": {"aId": 5744, "feed": "nvidia-csaf"}, +} + + +class Snap4VerdictTests(unittest.TestCase): + def setUp(self) -> None: + import tempfile + + table = json.loads(minimums.minimums_path().read_text()) + table["components"]["snap4Container"] = SNAP4_BLOCK + holder = tempfile.TemporaryDirectory() + self.addCleanup(holder.cleanup) + path = Path(holder.name) / "minimums.json" + path.write_text(json.dumps(table)) + previous = os.environ.get(minimums.MINIMUMS_ENV) + os.environ[minimums.MINIMUMS_ENV] = str(path) + minimums._CACHE = None + + def restore() -> None: + if previous is None: + os.environ.pop(minimums.MINIMUMS_ENV, None) + else: + os.environ[minimums.MINIMUMS_ENV] = previous + minimums._CACHE = None + + self.addCleanup(restore) + + def grade(self, version: str) -> tuple[str, str]: + verdict, _ = security.snap4_verdict(version) + return verdict.status, verdict.minimum + + def test_each_train_is_held_to_its_own_fix(self) -> None: + self.assertEqual(self.grade("4.9.0"), ("fail", "4.9.1")) + self.assertEqual(self.grade("4.9.1"), ("pass", "4.9.1")) + self.assertEqual(self.grade("4.5.4"), ("fail", "4.5.5")) + self.assertEqual(self.grade("4.5.5"), ("pass", "4.5.5")) + + def test_a_train_without_a_fix_moves_to_the_next_fix_above(self) -> None: + self.assertEqual(self.grade("4.7.2"), ("fail", "4.9.1")) + self.assertEqual(self.grade("4.3.0"), ("fail", "4.5.5")) + + def test_a_train_newer_than_every_fix_passes(self) -> None: + self.assertEqual(self.grade("4.12.0"), ("pass", "4.9.1")) + + def result(self, state: str, *observed: tuple[str, str], reason: str = "") -> dict: + return security.snap4_container_result( + { + "state": state, + "reason": reason, + "observed": [{"version": v, "host": h, "source": "rshim-ssh/crictl"} for v, h in observed], + } + ) + + def test_the_worst_reading_decides(self) -> None: + result = self.result("version", ("4.12.0", "a"), ("4.9.0", "b"), ("4.5.5", "c")) + self.assertEqual(result["status"], "fail") + self.assertEqual(result["version"], "4.9.0") + self.assertEqual(result["minimum"], "4.9.1") + self.assertEqual(result["host"], "b") + + def test_a_fail_stands_beside_an_unread_peer(self) -> None: + result = self.result("unknown", ("4.7.0", "a"), reason="DPU not reached") + self.assertEqual(result["status"], "fail") + self.assertIn("Not every DPU was read", result["detail"]) + + def test_passing_readings_do_not_clear_unread_peers(self) -> None: + result = self.result("unknown", ("4.9.1", "a"), reason="DPU not reached") + self.assertEqual(result["status"], "unknown") + self.assertIn("not every DPU was read", result["detail"]) + + def test_passing_readings_with_full_coverage_pass(self) -> None: + result = self.result("not_running", ("4.9.1", "a")) + self.assertEqual(result["status"], "pass") + + def test_nothing_to_read_is_not_applicable(self) -> None: + for state in ("not_applicable", "not_installed", "not_running"): + with self.subTest(state=state): + self.assertEqual(self.result(state)["status"], "not_applicable") + + def test_unread_dpu_mode_is_unknown(self) -> None: + result = self.result("unknown", reason="the DPU did not answer") + self.assertEqual(result["status"], "unknown") + self.assertIn("did not answer", result["detail"]) + + def test_no_rollup_is_unknown(self) -> None: + self.assertEqual(security.snap4_container_result(None)["status"], "unknown") + + +class Snap4WithoutTableEntryTests(unittest.TestCase): + """The shipped table predates snap4Container until the refresh publishes it.""" + + def test_reading_without_a_minimum_is_unknown_and_absence_still_applies(self) -> None: + table = json.loads(minimums.minimums_path().read_text()) + if "snap4Container" in table.get("components", {}): + self.skipTest("the shipped table already lists snap4Container") + read = security.snap4_container_result( + {"state": "version", "observed": [{"version": "4.9.1", "host": "a", "source": "x"}]} + ) + self.assertEqual(read["status"], "unknown") + absent = security.snap4_container_result({"state": "not_applicable", "observed": []}) + self.assertEqual(absent["status"], "not_applicable") + + +if __name__ == "__main__": + unittest.main() From e7de96a5b49408be29dfd18799368f33b13d2b0f Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:09:15 +0000 Subject: [PATCH 02/12] Title SNAP-4 rows as SNAP-4 in the audit report --- cmax/audit_report.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cmax/audit_report.py b/cmax/audit_report.py index f876658..04a26c5 100644 --- a/cmax/audit_report.py +++ b/cmax/audit_report.py @@ -302,6 +302,7 @@ def _title(key: str) -> str: "Rdma": "RDMA", "Hca": "HCA", "Dpu": "DPU", + "Snap4": "SNAP-4", "Dcgm": "DCGM", "Smmu": "SMMU", "Iommu": "IOMMU", From 6e4673dbd1cfa09f1b9bb9099f29cc5f4bdba09f Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:12:04 +0000 Subject: [PATCH 03/12] Keep the standalone reason in the SNAP-4 verdict and pin the collector flag --- cmax/scripts/1-audit/security_version_audit.py | 6 +++++- tests/audit/test_minimum_versions_shell.py | 4 ++++ tests/audit/test_security_version_audit.py | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cmax/scripts/1-audit/security_version_audit.py b/cmax/scripts/1-audit/security_version_audit.py index ef93252..93d0008 100644 --- a/cmax/scripts/1-audit/security_version_audit.py +++ b/cmax/scripts/1-audit/security_version_audit.py @@ -597,7 +597,11 @@ def finish(verdict: Verdict, selector: str | None, entry: dict[str, Any] | None) "not_applicable", minimum_text, advisory, - "No BlueField-3 DPU is present in the completed scan, so SNAP-4 does not apply", + # The standalone harness says why it skipped the scan; a fleet + # scan that found no BlueField-3 says that instead. + "No BlueField-3 DPU is present in the completed scan, so SNAP-4 does not apply" + if not reason or reason.startswith("no BlueField-3") + else f"SNAP-4 does not apply: {reason}", ) elif state == "not_installed": verdict = Verdict( diff --git a/tests/audit/test_minimum_versions_shell.py b/tests/audit/test_minimum_versions_shell.py index 39bc7ba..928207d 100644 --- a/tests/audit/test_minimum_versions_shell.py +++ b/tests/audit/test_minimum_versions_shell.py @@ -1658,6 +1658,7 @@ def test_every_virtio_net_flag_is_emitted_with_its_value(self) -> None: "virtioNetWorstObservedMode": "dpu", "virtioNetWorstObservedHost": "worker-3", "virtioNetObservedJson": '[{"host": "worker-3"}]', + "snap4Json": '{"state": "version", "observed": [{"version": "4.9.0"}]}', } ) @@ -1674,6 +1675,9 @@ def test_every_virtio_net_flag_is_emitted_with_its_value(self) -> None: self.assertEqual(pairs["--virtio-net-worst-host"], "worker-3") self.assertEqual(pairs["--virtio-net-observed-json"], '[{"host": "worker-3"}]') self.assertEqual(pairs["--dpu-isolation-json"], '{"scanComplete": true}') + self.assertEqual( + pairs["--snap4-json"], '{"state": "version", "observed": [{"version": "4.9.0"}]}' + ) class DegradedSecurityAuditFallbackTests(TempDirMixin): def evaluator_components(self) -> set[str]: diff --git a/tests/audit/test_security_version_audit.py b/tests/audit/test_security_version_audit.py index 60ae888..bc1fd16 100644 --- a/tests/audit/test_security_version_audit.py +++ b/tests/audit/test_security_version_audit.py @@ -1162,6 +1162,7 @@ class VirtioNetRollupReachesTheVerdictTests(unittest.TestCase): "virtioNetWorstObservedHost": "--virtio-net-worst-host", "virtioNetObservedJson": "--virtio-net-observed-json", "dpuIsolationJson": "--dpu-isolation-json", + "snap4Json": "--snap4-json", } @classmethod From bb4f7a65315ea52d4412121c110b061d76cae901 Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:27:21 +0000 Subject: [PATCH 04/12] virtio-net check: read PCI IDs from sysfs when lspci is missing Slurm-on-Kubernetes slurmd images (CoreWeave SUNK) ship without pciutils but mount the host sysfs, so the BlueField-3 scan, and the SNAP-4 scope that depends on it, were left unread. Fall back to /sys/bus/pci/devices and record pciSource on the host record. --- .../1-audit/checks/fabric/virtio-net-check.py | 40 +++++++++++++ tests/audit/test_virtio_net_check.py | 59 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py index 217802d..47b1f4f 100755 --- a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py +++ b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py @@ -476,10 +476,49 @@ def pci_scan_result( } +def sysfs_pci_listing(env: dict[str, str] = os.environ) -> tuple[str, str | None]: + """Synthesize an `lspci -Dn` style listing from ``/sys/bus/pci/devices``. + + Container images used as compute nodes (Slurm-on-Kubernetes slurmd pods, + for example) often ship without pciutils while the host's sysfs is still + mounted. The kernel's own vendor/device/class files answer the same + question lspci would, so they are read when lspci cannot run. Any entry + whose IDs cannot be read makes the listing partial: the devices that were + read still count as positive evidence, but absence is not claimed. + """ + root = Path(env.get("CLUSTERMAX_AUDIT_ROOT", "") + "/sys/bus/pci/devices") + try: + entries = sorted(root.iterdir()) + except OSError as exc: + return "", f"sysfs PCI tree unreadable: {exc.strerror or exc}" + lines: list[str] = [] + unreadable = 0 + for entry in entries: + try: + vendor = (entry / "vendor").read_text().strip().lower().removeprefix("0x") + device = (entry / "device").read_text().strip().lower().removeprefix("0x") + pci_class = (entry / "class").read_text().strip().lower().removeprefix("0x") + except OSError: + unreadable += 1 + continue + lines.append(f"{entry.name} {pci_class[:4]}: {vendor}:{device}") + if unreadable: + return "\n".join(lines), f"sysfs could not read {unreadable} PCI device(s)" + return "\n".join(lines), None + + def collect_pci( *, runner: Runner = subprocess.run, env: dict[str, str] = os.environ ) -> dict[str, Any]: text, error = run_tool(["lspci", "-Dnn"], runner=runner) + if error is not None: + sys_text, sys_error = sysfs_pci_listing(env) + if sys_text: + result = pci_scan_result(sys_text, sys_error) + result["pciSource"] = "sysfs" + if not result["scanComplete"]: + result["scanError"] = f"{error}; {sys_error}" + return result return pci_scan_result( text, error, empty_bus_confirmed=sysfs_pci_bus_empty(env) ) @@ -1267,6 +1306,7 @@ def gather( "host": env.get("NODE_NAME") or socket.gethostname(), "scanComplete": pci["scanComplete"], "scanError": pci["scanError"], + "pciSource": pci.get("pciSource", "lspci"), "bluefield3Present": bool(devices) if pci["scanComplete"] else None, "bluefieldDevices": devices, "virtioPciDevices": pci["virtioPciDevices"], diff --git a/tests/audit/test_virtio_net_check.py b/tests/audit/test_virtio_net_check.py index 0c9ca11..5dcfb7a 100644 --- a/tests/audit/test_virtio_net_check.py +++ b/tests/audit/test_virtio_net_check.py @@ -357,6 +357,65 @@ def test_missing_lspci_is_incomplete_not_not_applicable(self) -> None: self.assertFalse(record["scanComplete"]) self.assertIsNone(record["bluefield3Present"]) + def sysfs_device(self, slot: str, vendor: str, device: str, pci_class: str) -> Path: + path = self.root / "sys" / "bus" / "pci" / "devices" / slot + path.mkdir(parents=True) + (path / "vendor").write_text(f"0x{vendor}\n") + (path / "device").write_text(f"0x{device}\n") + (path / "class").write_text(f"0x{pci_class}\n") + return path + + def test_missing_lspci_falls_back_to_sysfs_and_finds_a_bluefield(self) -> None: + # Slurm-on-Kubernetes slurmd images ship without pciutils but still + # mount the host's sysfs, which names the same devices. + self.stub_missing("lspci") + self.stub_missing("mlxconfig") + self.sysfs_device("0000:9a:00.0", "15b3", "a2dc", "020000") + self.sysfs_device("0000:17:00.0", "15b3", "1021", "020700") + + record = self.host_record() + + self.assertTrue(record["scanComplete"]) + self.assertEqual(record["pciSource"], "sysfs") + self.assertTrue(record["bluefield3Present"]) + self.assertEqual(record["bluefieldDevices"][0]["slot"], "0000:9a:00.0") + self.assertEqual(record["state"], "incomplete") + + def test_missing_lspci_with_a_bluefield_free_sysfs_is_not_applicable(self) -> None: + self.stub_missing("lspci") + self.sysfs_device("0000:17:00.0", "15b3", "1021", "020700") + self.sysfs_device("0000:00:04.0", "1af4", "1001", "010000") + + record = self.host_record() + + self.assertTrue(record["scanComplete"]) + self.assertEqual(record["pciSource"], "sysfs") + self.assertFalse(record["bluefield3Present"]) + self.assertEqual(record["virtioPciDevices"], 1) + self.assertEqual(record["state"], "not_applicable") + + def test_an_unreadable_sysfs_entry_keeps_the_fallback_incomplete(self) -> None: + self.stub_missing("lspci") + self.sysfs_device("0000:17:00.0", "15b3", "1021", "020700") + broken = self.root / "sys" / "bus" / "pci" / "devices" / "0000:18:00.0" + broken.mkdir(parents=True) + + record = self.host_record() + + self.assertFalse(record["scanComplete"]) + self.assertIsNone(record["bluefield3Present"]) + self.assertIn("sysfs could not read 1", record["scanError"]) + self.assertEqual(record["state"], "incomplete") + + def test_lspci_stays_the_source_when_it_runs(self) -> None: + self.stub_output("lspci", NO_BLUEFIELD_LSPCI) + self.sysfs_device("0000:9a:00.0", "15b3", "a2dc", "020000") + + record = self.host_record() + + self.assertEqual(record["pciSource"], "lspci") + self.assertFalse(record["bluefield3Present"]) + def test_mlxconfig_without_root_is_incomplete_not_not_applicable(self) -> None: self.stub_output("lspci", BLUEFIELD3_LSPCI) From e3913ac18f06a405a239bb24a5a68e229517a3d3 Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:41:39 +0000 Subject: [PATCH 05/12] virtio-net check: do not blame sudo when mlxconfig is not installed On CoreWeave SUNK sudo -n works and mlxconfig is simply absent, but the reason said the sudo retry did not answer. Name the missing tool instead. --- .../1-audit/checks/fabric/virtio-net-check.py | 4 ++++ tests/audit/test_virtio_net_check.py | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py index 47b1f4f..aa449fb 100755 --- a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py +++ b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py @@ -1265,6 +1265,10 @@ def decide(record: dict[str, Any]) -> tuple[str, str]: ), ESCALATION_ROOT: "mlxconfig ran with root rights and still reported no mode", }.get(str(record.get("modeEscalation")), "mlxconfig needs root") + if "is not installed" in str(record.get("modeError") or ""): + # A missing tool is not a privilege problem. Saying sudo was + # refused here would send an operator after the wrong fix. + privilege = "install mlxconfig (NVIDIA MFT) or mstconfig to read it" return ( STATE_INCOMPLETE, "BlueField-3 is present but its mode could not be read " diff --git a/tests/audit/test_virtio_net_check.py b/tests/audit/test_virtio_net_check.py index 5dcfb7a..9cc1e98 100644 --- a/tests/audit/test_virtio_net_check.py +++ b/tests/audit/test_virtio_net_check.py @@ -520,6 +520,19 @@ def test_missing_mlxconfig_is_incomplete(self) -> None: self.assertEqual(record["state"], "incomplete") self.assertIn("mlxconfig", record["reason"]) + def test_missing_mlxconfig_is_not_blamed_on_sudo(self) -> None: + # Seen on CoreWeave SUNK: sudo -n works, mlxconfig is simply absent. + self.stub_output("lspci", BLUEFIELD3_LSPCI) + self.stub_missing("mlxconfig") + self.stub_sudo() + + record = self.host_record() + + self.assertEqual(record["state"], "incomplete") + self.assertIn("mlxconfig is not installed", record["reason"]) + self.assertIn("install mlxconfig", record["reason"]) + self.assertNotIn("sudo", record["reason"]) + class EscalationPathTests(unittest.TestCase): def setUp(self) -> None: From 5dfa5f56aae8f980ba4f2ef7c8b88e58ec6e558d Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Wed, 4 Nov 2026 20:45:49 +0000 Subject: [PATCH 06/12] virtio-net check: name only the tool the check actually runs --- cmax/scripts/1-audit/checks/fabric/virtio-net-check.py | 2 +- tests/audit/test_virtio_net_check.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py index aa449fb..29cd64e 100755 --- a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py +++ b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py @@ -1268,7 +1268,7 @@ def decide(record: dict[str, Any]) -> tuple[str, str]: if "is not installed" in str(record.get("modeError") or ""): # A missing tool is not a privilege problem. Saying sudo was # refused here would send an operator after the wrong fix. - privilege = "install mlxconfig (NVIDIA MFT) or mstconfig to read it" + privilege = "install NVIDIA MFT, which provides mlxconfig, to read it" return ( STATE_INCOMPLETE, "BlueField-3 is present but its mode could not be read " diff --git a/tests/audit/test_virtio_net_check.py b/tests/audit/test_virtio_net_check.py index 9cc1e98..20faf71 100644 --- a/tests/audit/test_virtio_net_check.py +++ b/tests/audit/test_virtio_net_check.py @@ -530,7 +530,7 @@ def test_missing_mlxconfig_is_not_blamed_on_sudo(self) -> None: self.assertEqual(record["state"], "incomplete") self.assertIn("mlxconfig is not installed", record["reason"]) - self.assertIn("install mlxconfig", record["reason"]) + self.assertIn("install NVIDIA MFT", record["reason"]) self.assertNotIn("sudo", record["reason"]) class EscalationPathTests(unittest.TestCase): From b7b3f807e6aa91674944f4b6bb62ba1c185eb10a Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:13:11 +0000 Subject: [PATCH 07/12] virtio-net check: query only the graded mlxconfig parameters The full mlxconfig -e -d q dump took 22 s on an idle GB300 BlueField-3 host (about 60 s with three reads in flight), past the 15 s tool timeout, so 14 of 16 VP hosts graded the mode unreadable. Naming INTERNAL_CPU_MODEL, INTERNAL_CPU_OFFLOAD_ENGINE and INTERNAL_CPU_RSHIM answers in under 2 s. Firmware that rejects a name falls back to the full dump once, under the privilege that opened the device, with a 60 s budget. --- .../1-audit/checks/fabric/virtio-net-check.py | 66 +++++++++++++++---- tests/audit/test_virtio_net_check.py | 60 +++++++++++++++++ 2 files changed, 114 insertions(+), 12 deletions(-) diff --git a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py index 29cd64e..5e71594 100755 --- a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py +++ b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py @@ -148,6 +148,9 @@ # Per-command timeout for every local tool. SLURM_TIMEOUT_S below is derived # from it, so the two move together. TOOL_TIMEOUT_S = 15 +# The full mlxconfig dump, read only when firmware rejects a named parameter. +# It took 22 s idle and about 60 s under contention on a GB300 BlueField-3. +MLXCONFIG_FULL_TIMEOUT_S = 60 # ssh gets ConnectTimeout=SSH_TIMEOUT_S and a wall-clock budget of twice that, # so a connection that opens and then stalls is still bounded. SSH_TIMEOUT_S = 8 @@ -159,20 +162,21 @@ # Worst case for one host, every command hanging until its own timeout: # # lspci 1 x 15 = 15 s -# mlxconfig q, sudo -n mlxconfig q, -# sudo -n mst start, sudo -n mlxconfig q 4 x 15 = 60 s +# named mlxconfig q, sudo -n named q 2 x 15 = 30 s +# full mlxconfig q, only after a named read +# answered "Unknown Parameter" 1 x 60 = 60 s # local `virtnet version` + `virtnet -v` 2 x 15 = 30 s # RShim SSH: version, -v, list, mlnx-release 4 x 16 = 64 s # SNAP-4 listing, one RShim SSH command 1 x 16 = 16 s # ------ -# 185 s +# 215 s # -# The local-virtnet-succeeded path and the RShim SSH path are mutually -# exclusive (collect_version only reaches SSH when the local read produced no -# version), and the succeeded path is cheaper: 15 + 60 + 15 + 2 x 15 + 15 = -# 135 s. So 185 s is a true ceiling, not a sum of alternatives. Allow 60 s for -# srun dispatch and worker startup, giving 245 s against a 300 s budget, about -# 18% margin. +# The full dump runs only after one of the named reads answered, so the mode +# read is at most 15 + 15 + 60 = 90 s. The local-virtnet-succeeded path and the +# RShim SSH path are mutually exclusive (collect_version only reaches SSH when +# the local read produced no version), and the succeeded path is cheaper. So +# 215 s is a true ceiling, not a sum of alternatives. Allow 60 s for srun +# dispatch and worker startup, giving 275 s against a 300 s budget. # # Redo this arithmetic when changing TOOL_TIMEOUT_S or SSH_TIMEOUT_S, or when # adding a command to gather(). An overrun is no longer a false clean, because @@ -571,6 +575,24 @@ def parse_mlxconfig(text: str) -> dict[str, str]: return values +def mlxconfig_query(slot: str, *, named: bool = True) -> list[str]: + """The mode read: only the parameters the check grades. + + The full `mlxconfig -e -d q` dump walks every parameter on the card. + On VP's GB300 BlueField-3 hosts it took 22 s on an idle host and about 60 s + with three reads in flight, past TOOL_TIMEOUT_S, so 14 of 16 hosts timed out + and graded the mode unreadable. Naming the three parameters answers in under + 2 s. ``named=False`` is the fallback for firmware that rejects a name. + """ + query = ["mlxconfig", "-e", "-d", slot, "q"] + return [*query, *_MLXCONFIG_KEYS] if named else query + + +def _unknown_parameter(text: str) -> bool: + """mlxconfig rejected a named parameter, so the device did open.""" + return "Unknown Parameter" in (text or "") + + def privileged_prefix( *, which: Which = shutil.which, @@ -635,18 +657,34 @@ def collect_mode( return result slot = devices[0].get("slot") or "" - query = ["mlxconfig", "-e", "-d", slot, "q"] + query = mlxconfig_query(slot) prefix, escalation = privileged_prefix(which=which, euid=euid) result["escalation"] = escalation source = "mlxconfig" text, error = run_tool(query, runner=runner) values = parse_mlxconfig(text) - if not values and prefix: + answered: list[str] | None = [] if values or _unknown_parameter(text) else None + if not values and answered is None and prefix: text, sudo_error = run_tool([*prefix, *query], runner=runner) values = parse_mlxconfig(text) error = error or sudo_error + if values or _unknown_parameter(text): + answered = prefix if values: source = "sudo mlxconfig" + if not values and answered is not None and _unknown_parameter(text): + # Firmware that predates one of the named parameters rejects the whole + # named query. Only then is the full dump read, once, under the + # privilege that already opened the device, with its own longer budget. + text, full_error = run_tool( + [*answered, *mlxconfig_query(slot, named=False)], + runner=runner, + timeout=MLXCONFIG_FULL_TIMEOUT_S, + ) + values = parse_mlxconfig(text) + error = full_error or error + if values: + source = "sudo mlxconfig" if answered else "mlxconfig" if not values: result["error"] = error or "mlxconfig returned no readable Current mode parameters" return result @@ -1828,7 +1866,11 @@ def slurm_records( printf '@@MODE_ERROR@@\t%s\n' "Multiple BlueField-3 PCI devices require per-device mode evidence" elif [ -n "$slot" ] && command -v mlxconfig >/dev/null 2>&1; then echo @@MLXCONFIG_BEGIN@@ - mlxconfig -e -d "$slot" q 2>&1 + mode_out=$(mlxconfig -e -d "$slot" q INTERNAL_CPU_MODEL INTERNAL_CPU_OFFLOAD_ENGINE INTERNAL_CPU_RSHIM 2>&1) + case "$mode_out" in + *"Unknown Parameter"*) mode_out=$(mlxconfig -e -d "$slot" q 2>&1) ;; + esac + printf '%s\n' "$mode_out" echo @@MLXCONFIG_END@@ elif [ -n "$slot" ]; then printf '@@MODE_ERROR@@\t%s\n' "mlxconfig is not installed" diff --git a/tests/audit/test_virtio_net_check.py b/tests/audit/test_virtio_net_check.py index 20faf71..cba8941 100644 --- a/tests/audit/test_virtio_net_check.py +++ b/tests/audit/test_virtio_net_check.py @@ -644,6 +644,66 @@ def test_failed_mode_read_never_loads_host_kernel_modules(self) -> None: self.assertFalse(self.calls_for("mst")) self.assertFalse(any(call.startswith("-n mst") for call in self.calls_for("sudo"))) +class NamedModeQueryTests(CheckHarness): + """The mode read names its three parameters instead of dumping the card. + + On VP's GB300 BlueField-3 hosts the full dump took 22 s idle, past the 15 s + tool timeout, and 14 of 16 hosts graded the mode unreadable. + """ + + def test_the_mode_read_names_only_the_graded_parameters(self) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + self.stub_output("mlxconfig", NIC_MODE_MLXCONFIG) + + record = self.host_record() + + self.assertEqual(record["mode"], "nic") + calls = self.calls_for("mlxconfig") + self.assertEqual(len(calls), 1) + self.assertTrue( + calls[0].endswith( + "q INTERNAL_CPU_MODEL INTERNAL_CPU_OFFLOAD_ENGINE INTERNAL_CPU_RSHIM" + ), + calls[0], + ) + + def test_firmware_that_rejects_a_name_falls_back_to_the_full_dump(self) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + data = self.tmp / "full.stdout" + data.write_text(NIC_MODE_MLXCONFIG) + self.stub( + "mlxconfig", + 'if [ "${@: -1}" = "q" ]; then /bin/cat "%s"; exit 0; fi\n' + "echo 'Configurations: Default Current Next Boot'\n" + "echo '-E- Unknown Parameter: INTERNAL_CPU_RSHIM'\n" + "exit 3" % data, + ) + self.stub_sudo() + + record = self.host_record() + + self.assertEqual(record["mode"], "nic") + self.assertEqual(record["modeSource"], "mlxconfig") + calls = self.calls_for("mlxconfig") + self.assertEqual(len(calls), 2) + self.assertTrue(calls[1].endswith(" q"), calls[1]) + # The device opened unprivileged, so nothing escalates. + self.assertEqual(self.calls_for("sudo"), []) + + def test_the_full_dump_is_never_read_after_a_failed_open(self) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + self.stub( + "mlxconfig", 'echo "-E- Failed to open the device" >&2\nexit 1' + ) + self.stub_sudo_denied() + + record = self.host_record() + + self.assertEqual(record["state"], "incomplete") + calls = self.calls_for("mlxconfig") + self.assertEqual(len(calls), 1) + self.assertFalse(calls[0].endswith(" q")) + class RshimEvidenceTests(CheckHarness): def test_rshim_reachability_is_recorded_as_evidence(self) -> None: From 2703f8c4482f2edfd531b10373f6dd1dfe49a484 Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:51:33 +0000 Subject: [PATCH 08/12] virtio-net check: also fall back when firmware lacks a named parameter MFT reports a parameter the firmware does not carry as "-E- The Device doesn't support X parameter" (seen on a ConnectX-8 with MFT 4.33), not "Unknown Parameter". Treat both as the device having opened, so older BlueField-3 firmware still reaches the full-dump fallback. --- .../1-audit/checks/fabric/virtio-net-check.py | 12 +++++++++--- tests/audit/test_virtio_net_check.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py index 5e71594..5f67815 100755 --- a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py +++ b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py @@ -589,8 +589,14 @@ def mlxconfig_query(slot: str, *, named: bool = True) -> list[str]: def _unknown_parameter(text: str) -> bool: - """mlxconfig rejected a named parameter, so the device did open.""" - return "Unknown Parameter" in (text or "") + """mlxconfig rejected a named parameter, so the device did open. + + MFT words this two ways: ``-E- Unknown Parameter: X`` for a name it does not + know, and ``-E- The Device doesn't support X parameter`` (seen on a + ConnectX-8 with MFT 4.33) for one the firmware lacks. + """ + text = text or "" + return "Unknown Parameter" in text or "doesn't support" in text def privileged_prefix( @@ -1868,7 +1874,7 @@ def slurm_records( echo @@MLXCONFIG_BEGIN@@ mode_out=$(mlxconfig -e -d "$slot" q INTERNAL_CPU_MODEL INTERNAL_CPU_OFFLOAD_ENGINE INTERNAL_CPU_RSHIM 2>&1) case "$mode_out" in - *"Unknown Parameter"*) mode_out=$(mlxconfig -e -d "$slot" q 2>&1) ;; + *"Unknown Parameter"*|*"support INTERNAL_CPU"*) mode_out=$(mlxconfig -e -d "$slot" q 2>&1) ;; esac printf '%s\n' "$mode_out" echo @@MLXCONFIG_END@@ diff --git a/tests/audit/test_virtio_net_check.py b/tests/audit/test_virtio_net_check.py index cba8941..aa045a9 100644 --- a/tests/audit/test_virtio_net_check.py +++ b/tests/audit/test_virtio_net_check.py @@ -690,6 +690,23 @@ def test_firmware_that_rejects_a_name_falls_back_to_the_full_dump(self) -> None: # The device opened unprivileged, so nothing escalates. self.assertEqual(self.calls_for("sudo"), []) + def test_a_parameter_the_firmware_lacks_also_falls_back(self) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + data = self.tmp / "full.stdout" + data.write_text(NIC_MODE_MLXCONFIG) + self.stub( + "mlxconfig", + 'if [ "${@: -1}" = "q" ]; then /bin/cat "%s"; exit 0; fi\n' + "echo 'Configurations: Next Boot'\n" + "echo \"-E- The Device doesn't support INTERNAL_CPU_RSHIM parameter\"\n" + "exit 3" % data, + ) + + record = self.host_record() + + self.assertEqual(record["mode"], "nic") + self.assertEqual(len(self.calls_for("mlxconfig")), 2) + def test_the_full_dump_is_never_read_after_a_failed_open(self) -> None: self.stub_output("lspci", BLUEFIELD3_LSPCI) self.stub( From 873f8e8343d2d7bdfab461b05723f58cec239b17 Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:00:09 +0000 Subject: [PATCH 09/12] dpu host isolation: warn, not fail, for NIC-mode RShim with the NIC-mode fix NVIDIA does not support the zero-trust (mlxprivhost restricted) profile in NIC mode ("Operating in NIC Mode with Zero Trust (Restricted) configuration is not supported and may lead to undefined behavior", BlueField BSP 4.10), yet a NIC-mode card with INTERNAL_CPU_RSHIM=0 graded fail and told the provider to run mlxprivhost r. The isolation evidence now carries platformMode, "nic" only when every host with an unrestricted RShim read NIC mode. In NIC mode with no open /dev/rshim0 or tmfifo_net0 the verdict is warn and recommends mlxconfig -d s INTERNAL_CPU_RSHIM=1 plus a power cycle; an open path still fails but names the NIC-mode fix. DPU or unread mode keeps the zero-trust grade. --- cmax/scripts/1-audit/audit_findings.py | 48 +++++++++++++++++-- .../1-audit/checks/fabric/virtio-net-check.py | 19 ++++++++ .../scripts/1-audit/security_version_audit.py | 48 +++++++++++++++++-- cmax/security.py | 11 +++++ tests/audit/test_audit_findings.py | 38 +++++++++++++++ tests/audit/test_security_version_audit.py | 48 +++++++++++++++++++ tests/audit/test_virtio_net_check.py | 32 +++++++++++++ 7 files changed, 237 insertions(+), 7 deletions(-) diff --git a/cmax/scripts/1-audit/audit_findings.py b/cmax/scripts/1-audit/audit_findings.py index b000c4c..a3c3629 100644 --- a/cmax/scripts/1-audit/audit_findings.py +++ b/cmax/scripts/1-audit/audit_findings.py @@ -875,6 +875,15 @@ def _tenant_pod_guidance( _status_is("fail"), harnesses=SCALE_OUT_HARNESSES, ), + Rule( + "securityVersions.dpuHostIsolation.status", + "BlueField in NIC mode leaves RShim open to the host " + "(INTERNAL_CPU_RSHIM=0); restrict it with mlxconfig " + "INTERNAL_CPU_RSHIM=1 and a power cycle", + CONFIG, + _status_is("warn"), + harnesses=SCALE_OUT_HARNESSES, + ), Rule( "securityVersions.dpuHostIsolation.status", "DPU host isolation could not be verified (mlxconfig usually needs " @@ -1934,10 +1943,43 @@ def _finding_guidance( "mlxprivhost -d r --disable_rshim --disable_tracer " "--disable_counter_rd --disable_port_owner" ) + nic_mode = ( + isinstance(isolation, dict) + and str(isolation.get("platformMode") or "").strip().lower() == "nic" + ) + if nic_mode: + # NVIDIA does not support the zero-trust profile in NIC mode. + recommendation = ( + "Restrict RShim on every affected BlueField with " + f"`{remediation}`. Do not apply the zero-trust mlxprivhost " + "profile in NIC mode; NVIDIA does not support it there. Then " + "verify that `INTERNAL_CPU_RSHIM=1`, `/dev/rshim0` is absent, " + "and `tmfifo_net0` is absent." + ) + else: + recommendation = ( + "Apply zero-trust mode on every affected BlueField with " + f"`{remediation}`. Then verify that `INTERNAL_CPU_RSHIM=1`, " + "`/dev/rshim0` is absent, and `tmfifo_net0` is absent." + ) + elif ( + rule.key == "securityVersions.dpuHostIsolation.status" + and str(value).strip().lower() == "warn" + ): + isolation = nested_get( + audit, "securityVersions", "dpuHostIsolation", default={} + ) + detected = ( + str(isolation.get("detail") or "").strip() + if isinstance(isolation, dict) + else "" + ) or "The BlueField is in NIC mode and RShim is not restricted." recommendation = ( - "Apply zero-trust mode on every affected BlueField with " - f"`{remediation}`. Then verify that `INTERNAL_CPU_RSHIM=1`, " - "`/dev/rshim0` is absent, and `tmfifo_net0` is absent." + "Set `INTERNAL_CPU_RSHIM=1` on every BlueField with " + "`mlxconfig -d s INTERNAL_CPU_RSHIM=1`, then power-cycle " + "the host so the firmware setting takes effect. Do not apply the " + "zero-trust mlxprivhost profile in NIC mode; NVIDIA does not support " + "it there. Recheck host isolation before switching any card to DPU mode." ) elif ( rule.key == "securityVersions.dpuHostIsolation.status" diff --git a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py index 5f67815..30da76d 100755 --- a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py +++ b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py @@ -1729,6 +1729,19 @@ def _isolation_proof_rung(record: dict[str, Any]) -> str: return ISO_REACHABLE if _isolation_reachable(record) else ISO_POSTURE_GAP +def _isolation_platform_mode(records: list[dict[str, Any]]) -> str: + unrestricted = [ + record + for record in records + if (record.get("rshimHostAccess") or {}).get("rshimRestricted") is False + or _isolation_path_reachable(record) + ] + if not unrestricted: + return "unknown" + modes = {str(record.get("mode") or "unknown").lower() for record in unrestricted} + return "nic" if modes == {"nic"} else ("dpu" if "dpu" in modes else "unknown") + + def isolation_evidence(records: list[dict[str, Any]]) -> dict[str, Any]: """The one host record the DPU host-isolation verdict is graded from. @@ -1751,6 +1764,12 @@ def isolation_evidence(records: list[dict[str, Any]]) -> dict[str, Any]: "bluefield3Present": chosen.get("bluefield3Present"), "modeError": chosen.get("modeError"), "rshimHostAccess": chosen.get("rshimHostAccess") or {}, + # NIC mode is host-trusted by design and NVIDIA does not support the + # zero-trust (mlxprivhost restricted) profile there, so the verdict + # grades an unrestricted RShim differently in NIC mode. It is "nic" only + # when every host whose RShim is unrestricted read NIC mode; one host in + # DPU mode, or one whose mode is unread, keeps the DPU-mode grade. + "platformMode": _isolation_platform_mode(records), "unassessedHosts": sorted( str(record.get("host") or "") for record in records diff --git a/cmax/scripts/1-audit/security_version_audit.py b/cmax/scripts/1-audit/security_version_audit.py index 93d0008..37d96e5 100644 --- a/cmax/scripts/1-audit/security_version_audit.py +++ b/cmax/scripts/1-audit/security_version_audit.py @@ -1007,6 +1007,15 @@ def _virtio_running_verdict(version: str, *, line: str | None = None) -> Verdict "mlxprivhost -d r --disable_rshim --disable_tracer " "--disable_counter_rd --disable_port_owner" ) +# NIC mode is host-trusted by design, and NVIDIA does not support the zero-trust +# (mlxprivhost restricted) profile there: "Operating in NIC Mode with Zero Trust +# (Restricted) configuration is not supported and may lead to undefined +# behavior" (BlueField BSP 4.10). The supported NIC-mode restriction is the +# INTERNAL_CPU_RSHIM firmware parameter, which takes effect after a power cycle. +NIC_MODE_RSHIM_REMEDIATION = ( + "mlxconfig -d s INTERNAL_CPU_RSHIM=1, then a cold reboot " + "(power cycle) of the host" +) DPU_ISOLATION_REQUIREMENT = ( "RShim restricted (INTERNAL_CPU_RSHIM=1), no /dev/rshim0, no tmfifo_net0" ) @@ -1507,7 +1516,7 @@ def _isolation_coverage_caveat(verdict: Verdict, unassessed: Any) -> Verdict: the detail it got before. """ names = [str(host).strip() for host in (unassessed or []) if str(host).strip()] - if verdict.status != "fail" or not names: + if verdict.status not in {"fail", "warn"} or not names: return verdict return _with_caveat( verdict, @@ -1558,6 +1567,13 @@ def dpu_host_isolation_verdict(evidence: dict[str, Any] | None = None) -> Verdic # `restricted is False` is deliberately NOT part of that proof: it comes # from mlxconfig reading INTERNAL_CPU_RSHIM, which needs the device the scan # finds, so it cannot be trusted ahead of the scan. It keeps its place below. + nic_mode = str(facts.get("platformMode") or "").strip().lower() == "nic" + remedy = ( + f"Restrict RShim with `{NIC_MODE_RSHIM_REMEDIATION}`; NVIDIA does not " + "support the zero-trust (mlxprivhost) profile in NIC mode." + if nic_mode + else f"Provider policy may require `{MLXPRIVHOST_REMEDIATION}`." + ) if reachable: verdict = Verdict( observed, @@ -1567,8 +1583,7 @@ def dpu_host_isolation_verdict(evidence: dict[str, Any] | None = None) -> Verdic f"The host exposes management interface paths ({', '.join(reachable)}), " f"which fails the zero-trust no-host-interface policy. This does not prove usable " f"RShim access or DPU login. Verify device permissions, SECURE_NIC_MODE, " - f"DROP_MODE and RShim ownership before remediation. Provider policy " - f"may require `{MLXPRIVHOST_REMEDIATION}`.", + f"DROP_MODE and RShim ownership before remediation. {remedy}", ) if not scan_complete: # Name the gap without weakening the grade. The path is proven; what @@ -1601,6 +1616,26 @@ def dpu_host_isolation_verdict(evidence: dict[str, Any] | None = None) -> Verdic "No BlueField DPU is present in the completed device scan", ) + if restricted is False and nic_mode: + # Host-trusted by design and no interface path is open, so this is a + # hardening gap, not a failed zero-trust policy: a host root can still + # load the RShim driver and open the channel. + return _isolation_coverage_caveat( + Verdict( + observed, + "warn", + requirement, + advisory, + "The BlueField is in NIC mode with INTERNAL_CPU_RSHIM=0, so the " + "firmware lets the host open RShim. No /dev/rshim0 or tmfifo_net0 " + "is present, so no RShim driver has opened it, but a host root " + "can load one. NIC mode is host-trusted by design and NVIDIA does " + "not support the zero-trust (mlxprivhost) profile in it. Restrict " + f"RShim with `{NIC_MODE_RSHIM_REMEDIATION}`, and recheck this " + "before switching to DPU mode.", + ), + facts.get("unassessedHosts"), + ) if restricted is False: return _isolation_coverage_caveat( Verdict( @@ -1667,7 +1702,12 @@ def dpu_host_isolation_result(evidence: dict[str, Any] | None = None) -> dict[st for host in (facts.get("unassessedHosts") or []) if str(host).strip() ], - "remediation": MLXPRIVHOST_REMEDIATION, + "platformMode": facts.get("platformMode") or "unknown", + "remediation": ( + NIC_MODE_RSHIM_REMEDIATION + if str(facts.get("platformMode") or "").strip().lower() == "nic" + else MLXPRIVHOST_REMEDIATION + ), } diff --git a/cmax/security.py b/cmax/security.py index 5c72fee..041cf81 100644 --- a/cmax/security.py +++ b/cmax/security.py @@ -1630,6 +1630,17 @@ def _dpu_host_isolation(audit: dict[str, Any]) -> tuple[str, str, str]: f"RShim restricted ({observed})", "The host cannot reach the DPU control plane over RShim.", ) + if status == "warn": + # NIC mode is host-trusted by design and NVIDIA does not support the + # zero-trust mlxprivhost profile there, so the supported fix differs. + return ( + WARNING, + f"NIC mode, RShim not restricted ({observed})", + "The BlueField is in NIC mode and its firmware lets the host open " + "RShim, though no RShim driver has opened it. Set " + "`mlxconfig -d s INTERNAL_CPU_RSHIM=1` and power-cycle the " + "host; do not apply mlxprivhost zero-trust in NIC mode.", + ) if status == "fail": return ( CRITICAL, diff --git a/tests/audit/test_audit_findings.py b/tests/audit/test_audit_findings.py index e4a0b6c..571350b 100644 --- a/tests/audit/test_audit_findings.py +++ b/tests/audit/test_audit_findings.py @@ -1180,6 +1180,44 @@ def test_reachable_dpu_control_plane_names_the_remediation(self) -> None: self.assertIn("mlxprivhost -d 03:00.0", finding.recommendation) self.assertIn("INTERNAL_CPU_RSHIM=1", finding.recommendation) + def test_nic_mode_rshim_warning_recommends_the_nic_mode_fix(self) -> None: + keyed = by_key( + self.detect( + { + "dpuHostIsolation": { + "status": "warn", + "bluefieldPresent": True, + "platformMode": "nic", + "detail": "The BlueField is in NIC mode with INTERNAL_CPU_RSHIM=0.", + } + } + ) + ) + finding = keyed["securityVersions.dpuHostIsolation.status"] + self.assertEqual(finding.severity, audit_findings.CONFIG) + self.assertIn("NIC mode", finding.title) + self.assertNotIn("mlxprivhost r", finding.title) + self.assertIn("INTERNAL_CPU_RSHIM=1", finding.recommendation) + self.assertIn("power-cycle", finding.recommendation) + self.assertIn("Do not apply the zero-trust", finding.recommendation) + + def test_open_rshim_path_in_nic_mode_recommends_the_nic_mode_fix(self) -> None: + keyed = by_key( + self.detect( + { + "dpuHostIsolation": { + "status": "fail", + "bluefieldPresent": True, + "platformMode": "nic", + "remediation": "mlxconfig -d s INTERNAL_CPU_RSHIM=1", + } + } + ) + ) + finding = keyed["securityVersions.dpuHostIsolation.status"] + self.assertIn("Restrict RShim", finding.recommendation) + self.assertIn("Do not apply the zero-trust", finding.recommendation) + def test_perf_controls_name_values_and_persistent_actions(self) -> None: keyed = by_key( audit_findings.detect_findings( diff --git a/tests/audit/test_security_version_audit.py b/tests/audit/test_security_version_audit.py index bc1fd16..2b2eae6 100644 --- a/tests/audit/test_security_version_audit.py +++ b/tests/audit/test_security_version_audit.py @@ -1511,6 +1511,54 @@ def test_each_host_side_path_to_the_dpu_control_plane_fails(self) -> None: self.assertIn("mlxprivhost", verdict.detail) self.assertIn("--disable_rshim", verdict.detail) + def test_nic_mode_with_rshim_enabled_warns_with_the_nic_mode_fix(self) -> None: + # NVIDIA does not support the zero-trust (mlxprivhost) profile in NIC + # mode, so the finding must not tell the provider to apply it there. + evidence = { + **self.evidence(rshimRestricted=False, internalCpuRshim="0"), + "platformMode": "nic", + } + verdict = security.dpu_host_isolation_verdict(evidence) + self.assertEqual(verdict.status, "warn") + self.assertIn("INTERNAL_CPU_RSHIM=1", verdict.detail) + self.assertIn("power cycle", verdict.detail) + self.assertIn("does not support the zero-trust", verdict.detail) + self.assertNotIn("--disable_rshim", verdict.detail) + result = security.dpu_host_isolation_result(evidence) + self.assertEqual(result["remediation"], security.NIC_MODE_RSHIM_REMEDIATION) + self.assertEqual(result["platformMode"], "nic") + + def test_an_open_rshim_path_in_nic_mode_still_fails(self) -> None: + evidence = { + **self.evidence(rshimRestricted=False, rshimDeviceNode=True), + "platformMode": "nic", + } + verdict = security.dpu_host_isolation_verdict(evidence) + self.assertEqual(verdict.status, "fail") + self.assertIn("INTERNAL_CPU_RSHIM=1", verdict.detail) + self.assertNotIn("--disable_rshim", verdict.detail) + + def test_dpu_or_unknown_mode_with_rshim_enabled_still_fails(self) -> None: + for mode in ("dpu", "unknown", None): + with self.subTest(mode=mode): + evidence = { + **self.evidence(rshimRestricted=False, internalCpuRshim="0"), + "platformMode": mode, + } + verdict = security.dpu_host_isolation_verdict(evidence) + self.assertEqual(verdict.status, "fail") + self.assertIn("mlxprivhost", verdict.detail) + + def test_a_nic_mode_warning_names_unassessed_hosts(self) -> None: + evidence = { + **self.evidence(rshimRestricted=False, internalCpuRshim="0"), + "platformMode": "nic", + "unassessedHosts": ["worker-9"], + } + verdict = security.dpu_host_isolation_verdict(evidence) + self.assertEqual(verdict.status, "warn") + self.assertIn("worker-9", verdict.detail) + def test_a_missing_pci_scan_cannot_erase_a_proven_rshim_path(self) -> None: for label, paths in ( ("device node", {"rshimDeviceNode": True, "tmfifoNet0": False}), diff --git a/tests/audit/test_virtio_net_check.py b/tests/audit/test_virtio_net_check.py index aa045a9..5fd94cc 100644 --- a/tests/audit/test_virtio_net_check.py +++ b/tests/audit/test_virtio_net_check.py @@ -1680,3 +1680,35 @@ def _write_severity_matrix() -> None: _write_severity_matrix() else: unittest.main() + + +class IsolationPlatformModeTests(unittest.TestCase): + """The isolation evidence says NIC only when every unrestricted host is NIC.""" + + @staticmethod + def record(host: str, mode: str, restricted: bool) -> dict: + return { + "host": host, + "mode": mode, + "scanComplete": True, + "bluefield3Present": True, + "rshimHostAccess": { + "rshimRestricted": restricted, + "rshimDeviceNode": False, + "tmfifoNet0": False, + }, + } + + def test_modes(self) -> None: + mod = load_check_module() + cases = { + "all nic": ([("a", "nic", False), ("b", "nic", False)], "nic"), + "one dpu": ([("a", "nic", False), ("b", "dpu", False)], "dpu"), + "one unread": ([("a", "nic", False), ("b", "unknown", False)], "unknown"), + "restricted dpu ignored": ([("a", "nic", False), ("b", "dpu", True)], "nic"), + } + for label, (rows, expected) in cases.items(): + with self.subTest(case=label): + records = [self.record(*row) for row in rows] + self.assertEqual(mod.isolation_evidence(records)["platformMode"], expected) + From 3ee691cfc999a5f9894374fb37b0b1f785ebd044 Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:15:03 +0000 Subject: [PATCH 10/12] dpu host isolation: NIC-mode RShim is a fail again, keeping the NIC-mode fix A host root can open RShim on a NIC-mode BlueField-3 with INTERNAL_CPU_RSHIM=0: on VP the RShim PCIe function (0xc2d5) is exposed and unbound, mlxprivhost reports PRIVILEGED with disable_rshim FALSE, and NVIDIA's rshim user-space driver needs no kernel module. Through it a tenant reaches the DPU boot and console interface and can push a boot stream that outlives the tenancy, so the warning understated it. The status goes back to fail. What stays from the previous commit is the remediation: NIC mode names mlxconfig INTERNAL_CPU_RSHIM=1 plus a power cycle instead of mlxprivhost, which NVIDIA does not support in NIC mode. --- cmax/scripts/1-audit/audit_findings.py | 36 +++++++------------ .../scripts/1-audit/security_version_audit.py | 23 ++++++------ cmax/security.py | 13 +++---- tests/audit/test_audit_findings.py | 9 ++--- tests/audit/test_security_version_audit.py | 9 ++--- 5 files changed, 43 insertions(+), 47 deletions(-) diff --git a/cmax/scripts/1-audit/audit_findings.py b/cmax/scripts/1-audit/audit_findings.py index a3c3629..526adf6 100644 --- a/cmax/scripts/1-audit/audit_findings.py +++ b/cmax/scripts/1-audit/audit_findings.py @@ -507,6 +507,12 @@ def _bluefield_observed(audit: dict[str, Any]) -> bool: ) +def _isolation_nic_mode(audit: dict[str, Any]) -> bool: + """True when every host with an unrestricted RShim read NIC mode.""" + mode = nested_get(audit, "securityVersions", "dpuHostIsolation", "platformMode") + return str(mode or "").strip().lower() == "nic" + + def _bluefield_absence_proven(audit: dict[str, Any]) -> bool: """True only when a completed inventory found no BlueField device.""" isolation = nested_get(audit, "securityVersions", "dpuHostIsolation") @@ -873,15 +879,18 @@ def _tenant_pod_guidance( "--disable_counter_rd --disable_port_owner", CONFIG, _status_is("fail"), + guard=lambda audit: not _isolation_nic_mode(audit), harnesses=SCALE_OUT_HARNESSES, ), Rule( "securityVersions.dpuHostIsolation.status", - "BlueField in NIC mode leaves RShim open to the host " - "(INTERNAL_CPU_RSHIM=0); restrict it with mlxconfig " - "INTERNAL_CPU_RSHIM=1 and a power cycle", + "Host side can open RShim to the BlueField (NIC mode, " + "INTERNAL_CPU_RSHIM=0); restrict it with mlxconfig " + "INTERNAL_CPU_RSHIM=1 and a power cycle, not mlxprivhost, which " + "NVIDIA does not support in NIC mode", CONFIG, - _status_is("warn"), + _status_is("fail"), + guard=_isolation_nic_mode, harnesses=SCALE_OUT_HARNESSES, ), Rule( @@ -1962,25 +1971,6 @@ def _finding_guidance( f"`{remediation}`. Then verify that `INTERNAL_CPU_RSHIM=1`, " "`/dev/rshim0` is absent, and `tmfifo_net0` is absent." ) - elif ( - rule.key == "securityVersions.dpuHostIsolation.status" - and str(value).strip().lower() == "warn" - ): - isolation = nested_get( - audit, "securityVersions", "dpuHostIsolation", default={} - ) - detected = ( - str(isolation.get("detail") or "").strip() - if isinstance(isolation, dict) - else "" - ) or "The BlueField is in NIC mode and RShim is not restricted." - recommendation = ( - "Set `INTERNAL_CPU_RSHIM=1` on every BlueField with " - "`mlxconfig -d s INTERNAL_CPU_RSHIM=1`, then power-cycle " - "the host so the firmware setting takes effect. Do not apply the " - "zero-trust mlxprivhost profile in NIC mode; NVIDIA does not support " - "it there. Recheck host isolation before switching any card to DPU mode." - ) elif ( rule.key == "securityVersions.dpuHostIsolation.status" and str(value).strip().lower() == "unknown" diff --git a/cmax/scripts/1-audit/security_version_audit.py b/cmax/scripts/1-audit/security_version_audit.py index 37d96e5..793cf7a 100644 --- a/cmax/scripts/1-audit/security_version_audit.py +++ b/cmax/scripts/1-audit/security_version_audit.py @@ -1516,7 +1516,7 @@ def _isolation_coverage_caveat(verdict: Verdict, unassessed: Any) -> Verdict: the detail it got before. """ names = [str(host).strip() for host in (unassessed or []) if str(host).strip()] - if verdict.status not in {"fail", "warn"} or not names: + if verdict.status != "fail" or not names: return verdict return _with_caveat( verdict, @@ -1617,22 +1617,25 @@ def dpu_host_isolation_verdict(evidence: dict[str, Any] | None = None) -> Verdic ) if restricted is False and nic_mode: - # Host-trusted by design and no interface path is open, so this is a - # hardening gap, not a failed zero-trust policy: a host root can still - # load the RShim driver and open the channel. + # NIC mode is host-trusted by design, but a tenant with host root can + # still load the RShim driver and open the channel: the RShim PCIe + # function is exposed and nothing in firmware blocks it. Through it the + # tenant can push a boot stream that outlives their tenancy, so this + # stays a fail. Only the fix differs, because NVIDIA does not support the + # zero-trust (mlxprivhost) profile in NIC mode. return _isolation_coverage_caveat( Verdict( observed, - "warn", + "fail", requirement, advisory, "The BlueField is in NIC mode with INTERNAL_CPU_RSHIM=0, so the " "firmware lets the host open RShim. No /dev/rshim0 or tmfifo_net0 " - "is present, so no RShim driver has opened it, but a host root " - "can load one. NIC mode is host-trusted by design and NVIDIA does " - "not support the zero-trust (mlxprivhost) profile in it. Restrict " - f"RShim with `{NIC_MODE_RSHIM_REMEDIATION}`, and recheck this " - "before switching to DPU mode.", + "is present, so no RShim driver has opened it yet, but a host root " + "can load one and reach the DPU boot and console interface. No DPU " + "login or escape was attempted. NVIDIA does not support the " + "zero-trust (mlxprivhost) profile in NIC mode, so restrict RShim " + f"with `{NIC_MODE_RSHIM_REMEDIATION}`.", ), facts.get("unassessedHosts"), ) diff --git a/cmax/security.py b/cmax/security.py index 041cf81..9afeb71 100644 --- a/cmax/security.py +++ b/cmax/security.py @@ -1630,14 +1630,15 @@ def _dpu_host_isolation(audit: dict[str, Any]) -> tuple[str, str, str]: f"RShim restricted ({observed})", "The host cannot reach the DPU control plane over RShim.", ) - if status == "warn": - # NIC mode is host-trusted by design and NVIDIA does not support the - # zero-trust mlxprivhost profile there, so the supported fix differs. + if status == "fail" and str(verdict.get("platformMode") or "").lower() == "nic": + # NVIDIA does not support the zero-trust mlxprivhost profile in NIC + # mode, so the fix is the INTERNAL_CPU_RSHIM firmware restriction. return ( - WARNING, - f"NIC mode, RShim not restricted ({observed})", + CRITICAL, + f"RShim reachable from the host, NIC mode ({observed})", "The BlueField is in NIC mode and its firmware lets the host open " - "RShim, though no RShim driver has opened it. Set " + "RShim, so a tenant with host root can load the RShim driver and " + "reach the DPU boot and console interface. Set " "`mlxconfig -d s INTERNAL_CPU_RSHIM=1` and power-cycle the " "host; do not apply mlxprivhost zero-trust in NIC mode.", ) diff --git a/tests/audit/test_audit_findings.py b/tests/audit/test_audit_findings.py index 571350b..6a2f87e 100644 --- a/tests/audit/test_audit_findings.py +++ b/tests/audit/test_audit_findings.py @@ -1180,15 +1180,16 @@ def test_reachable_dpu_control_plane_names_the_remediation(self) -> None: self.assertIn("mlxprivhost -d 03:00.0", finding.recommendation) self.assertIn("INTERNAL_CPU_RSHIM=1", finding.recommendation) - def test_nic_mode_rshim_warning_recommends_the_nic_mode_fix(self) -> None: + def test_nic_mode_rshim_fail_names_the_nic_mode_fix(self) -> None: keyed = by_key( self.detect( { "dpuHostIsolation": { - "status": "warn", + "status": "fail", "bluefieldPresent": True, "platformMode": "nic", "detail": "The BlueField is in NIC mode with INTERNAL_CPU_RSHIM=0.", + "remediation": "mlxconfig -d s INTERNAL_CPU_RSHIM=1", } } ) @@ -1196,9 +1197,9 @@ def test_nic_mode_rshim_warning_recommends_the_nic_mode_fix(self) -> None: finding = keyed["securityVersions.dpuHostIsolation.status"] self.assertEqual(finding.severity, audit_findings.CONFIG) self.assertIn("NIC mode", finding.title) - self.assertNotIn("mlxprivhost r", finding.title) + self.assertIn("INTERNAL_CPU_RSHIM=1", finding.title) + self.assertNotIn("--disable_rshim", finding.title) self.assertIn("INTERNAL_CPU_RSHIM=1", finding.recommendation) - self.assertIn("power-cycle", finding.recommendation) self.assertIn("Do not apply the zero-trust", finding.recommendation) def test_open_rshim_path_in_nic_mode_recommends_the_nic_mode_fix(self) -> None: diff --git a/tests/audit/test_security_version_audit.py b/tests/audit/test_security_version_audit.py index 2b2eae6..6b09c39 100644 --- a/tests/audit/test_security_version_audit.py +++ b/tests/audit/test_security_version_audit.py @@ -1511,7 +1511,7 @@ def test_each_host_side_path_to_the_dpu_control_plane_fails(self) -> None: self.assertIn("mlxprivhost", verdict.detail) self.assertIn("--disable_rshim", verdict.detail) - def test_nic_mode_with_rshim_enabled_warns_with_the_nic_mode_fix(self) -> None: + def test_nic_mode_with_rshim_enabled_fails_with_the_nic_mode_fix(self) -> None: # NVIDIA does not support the zero-trust (mlxprivhost) profile in NIC # mode, so the finding must not tell the provider to apply it there. evidence = { @@ -1519,7 +1519,8 @@ def test_nic_mode_with_rshim_enabled_warns_with_the_nic_mode_fix(self) -> None: "platformMode": "nic", } verdict = security.dpu_host_isolation_verdict(evidence) - self.assertEqual(verdict.status, "warn") + # A host root can still load the RShim driver and open the channel. + self.assertEqual(verdict.status, "fail") self.assertIn("INTERNAL_CPU_RSHIM=1", verdict.detail) self.assertIn("power cycle", verdict.detail) self.assertIn("does not support the zero-trust", verdict.detail) @@ -1549,14 +1550,14 @@ def test_dpu_or_unknown_mode_with_rshim_enabled_still_fails(self) -> None: self.assertEqual(verdict.status, "fail") self.assertIn("mlxprivhost", verdict.detail) - def test_a_nic_mode_warning_names_unassessed_hosts(self) -> None: + def test_a_nic_mode_fail_names_unassessed_hosts(self) -> None: evidence = { **self.evidence(rshimRestricted=False, internalCpuRshim="0"), "platformMode": "nic", "unassessedHosts": ["worker-9"], } verdict = security.dpu_host_isolation_verdict(evidence) - self.assertEqual(verdict.status, "warn") + self.assertEqual(verdict.status, "fail") self.assertIn("worker-9", verdict.detail) def test_a_missing_pci_scan_cannot_erase_a_proven_rshim_path(self) -> None: From adf39033948d3e9a9bdb1b51ae81b9a73d217f49 Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:14:17 +0000 Subject: [PATCH 11/12] dpu host isolation: name all three provider-side fixes for NIC-mode RShim A tenant with host root can undo INTERNAL_CPU_RSHIM, so the two-way recommendation was incomplete: where you cannot block that access, re-image the card from a known-good BFB per tenant. Where you can, keep tenant host root off the BlueField while it is in NIC mode. Where you want the host gone entirely, run DPU mode with mlxprivhost restricted, which NVIDIA supports there. --- cmax/scripts/1-audit/audit_findings.py | 18 ++++++++++++------ cmax/scripts/1-audit/security_version_audit.py | 17 ++++++++++++++++- cmax/security.py | 8 +++++--- tests/audit/test_audit_findings.py | 12 +++++++----- tests/audit/test_security_version_audit.py | 3 +++ 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/cmax/scripts/1-audit/audit_findings.py b/cmax/scripts/1-audit/audit_findings.py index 526adf6..c23581c 100644 --- a/cmax/scripts/1-audit/audit_findings.py +++ b/cmax/scripts/1-audit/audit_findings.py @@ -885,9 +885,9 @@ def _tenant_pod_guidance( Rule( "securityVersions.dpuHostIsolation.status", "Host side can open RShim to the BlueField (NIC mode, " - "INTERNAL_CPU_RSHIM=0); restrict it with mlxconfig " - "INTERNAL_CPU_RSHIM=1 and a power cycle, not mlxprivhost, which " - "NVIDIA does not support in NIC mode", + "INTERNAL_CPU_RSHIM=0); keep tenant host root off the card or " + "re-image it per tenant, not mlxprivhost, which NVIDIA does not " + "support in NIC mode", CONFIG, _status_is("fail"), guard=_isolation_nic_mode, @@ -1959,9 +1959,15 @@ def _finding_guidance( if nic_mode: # NVIDIA does not support the zero-trust profile in NIC mode. recommendation = ( - "Restrict RShim on every affected BlueField with " - f"`{remediation}`. Do not apply the zero-trust mlxprivhost " - "profile in NIC mode; NVIDIA does not support it there. Then " + "Two ways to make this real isolation, not just a setting a " + "tenant can undo. Either keep tenant host root off the " + "BlueField (sudo -M, mdev-coldplug, or an IOMMU/ACS isolation " + "of the PCI function), so only the provider can reach it. Or " + "accept that a tenant can own the card and re-image it from a " + "known-good BFB before each new tenant and after each one " + "leaves. `mlxconfig -d s INTERNAL_CPU_RSHIM=1` plus a " + "power cycle is real only under the first path; NVIDIA does not " + "support the zero-trust mlxprivhost profile in NIC mode. Then " "verify that `INTERNAL_CPU_RSHIM=1`, `/dev/rshim0` is absent, " "and `tmfifo_net0` is absent." ) diff --git a/cmax/scripts/1-audit/security_version_audit.py b/cmax/scripts/1-audit/security_version_audit.py index 793cf7a..6f5f8fe 100644 --- a/cmax/scripts/1-audit/security_version_audit.py +++ b/cmax/scripts/1-audit/security_version_audit.py @@ -1016,6 +1016,17 @@ def _virtio_running_verdict(version: str, *, line: str | None = None) -> Verdict "mlxconfig -d s INTERNAL_CPU_RSHIM=1, then a cold reboot " "(power cycle) of the host" ) +# INTERNAL_CPU_RSHIM is itself just a card setting, so a tenant with host root +# can set it back. It is real isolation only when the host that gets sudo is the +# only one that can reach the card. Where a tenant has that privilege, the two +# real fixes are on the provider side: keep the tenant's host root off the +# BlueField (sudo -M, mdev-coldplug or an IOMMU/ACS isolation of the PCI +# function), or re-image the card from a known-good BFB before each new tenant +# and after each one leaves. +NIC_MODE_TENANT_SUDO_REMEDIATION = ( + "Keep tenant host root off the BlueField, or re-image the card from a " + "known-good BFB before each new tenant and after each one leaves" +) DPU_ISOLATION_REQUIREMENT = ( "RShim restricted (INTERNAL_CPU_RSHIM=1), no /dev/rshim0, no tmfifo_net0" ) @@ -1635,7 +1646,11 @@ def dpu_host_isolation_verdict(evidence: dict[str, Any] | None = None) -> Verdic "can load one and reach the DPU boot and console interface. No DPU " "login or escape was attempted. NVIDIA does not support the " "zero-trust (mlxprivhost) profile in NIC mode, so restrict RShim " - f"with `{NIC_MODE_RSHIM_REMEDIATION}`.", + f"with `{NIC_MODE_RSHIM_REMEDIATION}`. That only isolates the " + "card if a tenant's host root cannot re-open it: the setting is " + "card state, so a tenant with the same privilege can set it " + "back. Where tenants get host root, use " + f"`{NIC_MODE_TENANT_SUDO_REMEDIATION}` instead.", ), facts.get("unassessedHosts"), ) diff --git a/cmax/security.py b/cmax/security.py index 9afeb71..729f818 100644 --- a/cmax/security.py +++ b/cmax/security.py @@ -1638,9 +1638,11 @@ def _dpu_host_isolation(audit: dict[str, Any]) -> tuple[str, str, str]: f"RShim reachable from the host, NIC mode ({observed})", "The BlueField is in NIC mode and its firmware lets the host open " "RShim, so a tenant with host root can load the RShim driver and " - "reach the DPU boot and console interface. Set " - "`mlxconfig -d s INTERNAL_CPU_RSHIM=1` and power-cycle the " - "host; do not apply mlxprivhost zero-trust in NIC mode.", + "reach the DPU boot and console interface. Keep tenant host root off " + "the BlueField, or re-image the card from a known-good BFB per " + "tenant; `mlxconfig -d s INTERNAL_CPU_RSHIM=1` and a power " + "cycle is real only when a tenant cannot set it back. Do not apply " + "mlxprivhost zero-trust in NIC mode.", ) if status == "fail": return ( diff --git a/tests/audit/test_audit_findings.py b/tests/audit/test_audit_findings.py index 6a2f87e..1e59378 100644 --- a/tests/audit/test_audit_findings.py +++ b/tests/audit/test_audit_findings.py @@ -1197,10 +1197,12 @@ def test_nic_mode_rshim_fail_names_the_nic_mode_fix(self) -> None: finding = keyed["securityVersions.dpuHostIsolation.status"] self.assertEqual(finding.severity, audit_findings.CONFIG) self.assertIn("NIC mode", finding.title) - self.assertIn("INTERNAL_CPU_RSHIM=1", finding.title) + self.assertIn("re-image", finding.title) self.assertNotIn("--disable_rshim", finding.title) - self.assertIn("INTERNAL_CPU_RSHIM=1", finding.recommendation) - self.assertIn("Do not apply the zero-trust", finding.recommendation) + self.assertIn("re-image it from a known-good BFB", finding.recommendation) + self.assertIn("off the BlueField", finding.recommendation) + self.assertIn("does not support the zero-trust", finding.recommendation) + self.assertIn("real only under the first path", finding.recommendation) def test_open_rshim_path_in_nic_mode_recommends_the_nic_mode_fix(self) -> None: keyed = by_key( @@ -1216,8 +1218,8 @@ def test_open_rshim_path_in_nic_mode_recommends_the_nic_mode_fix(self) -> None: ) ) finding = keyed["securityVersions.dpuHostIsolation.status"] - self.assertIn("Restrict RShim", finding.recommendation) - self.assertIn("Do not apply the zero-trust", finding.recommendation) + self.assertIn("re-image it from a known-good BFB", finding.recommendation) + self.assertIn("real only under the first path", finding.recommendation) def test_perf_controls_name_values_and_persistent_actions(self) -> None: keyed = by_key( diff --git a/tests/audit/test_security_version_audit.py b/tests/audit/test_security_version_audit.py index 6b09c39..60c16db 100644 --- a/tests/audit/test_security_version_audit.py +++ b/tests/audit/test_security_version_audit.py @@ -1525,6 +1525,9 @@ def test_nic_mode_with_rshim_enabled_fails_with_the_nic_mode_fix(self) -> None: self.assertIn("power cycle", verdict.detail) self.assertIn("does not support the zero-trust", verdict.detail) self.assertNotIn("--disable_rshim", verdict.detail) + self.assertIn("can set it back", verdict.detail) + self.assertIn("re-image the card", verdict.detail) + self.assertIn("off the BlueField", verdict.detail) result = security.dpu_host_isolation_result(evidence) self.assertEqual(result["remediation"], security.NIC_MODE_RSHIM_REMEDIATION) self.assertEqual(result["platformMode"], "nic") From e5105535ebeb6f71c52c402c0c36d1b55013cd87 Mon Sep 17 00:00:00 2001 From: aliencaocao <20109683+aliencaocao@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:13:18 +0000 Subject: [PATCH 12/12] dpu host isolation: give mode-specific remediation in the check fix Ports the provider-remediation part of the private review (Jordan): the static fix no longer prescribes mlxprivhost zero-trust in NIC mode, which NVIDIA does not support. In NIC mode the provider must enforce RShim isolation or re-image from a trusted BFB between tenants. --- cmax/security.py | 10 +++++++--- tests/audit/test_security_cli.py | 6 ++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cmax/security.py b/cmax/security.py index 729f818..31eb579 100644 --- a/cmax/security.py +++ b/cmax/security.py @@ -2307,9 +2307,13 @@ def _observability(audit: dict[str, Any]) -> tuple[str, str, str]: "RShim is enabled by default, so the host reaches the DPU Arm cores " "directly. On a multi-tenant cluster a tenant with host root then " "reaches a control plane outside the hypervisor boundary.", - "Apply the zero-trust host profile: `mlxprivhost -d r " - "--disable_rshim --disable_tracer --disable_counter_rd " - "--disable_port_owner`.", + # NVIDIA does not support the zero-trust profile in NIC mode, so the + # static fix names both modes, matching the per-verdict assessment. + "In DPU mode, apply the zero-trust host profile: `mlxprivhost -d " + " r --disable_rshim --disable_tracer --disable_counter_rd " + "--disable_port_owner`. In NIC mode, do not apply mlxprivhost " + "zero-trust. The provider must enforce RShim isolation so tenant root " + "cannot reverse it, or re-image from a trusted BFB between tenants.", _references( ( "NVIDIA mlxprivhost host restriction", diff --git a/tests/audit/test_security_cli.py b/tests/audit/test_security_cli.py index ed2a33b..2365478 100644 --- a/tests/audit/test_security_cli.py +++ b/tests/audit/test_security_cli.py @@ -1322,6 +1322,12 @@ def test_a_present_device_still_passes_on_its_own_merits(self) -> None: ) self.assertEqual(status, security.PASS) + def test_remediation_does_not_prescribe_zero_trust_in_nic_mode(self) -> None: + spec = next(s for s in security.CHECK_SPECS if s.id == "dpu-host-isolation") + self.assertIn("In DPU mode, apply the zero-trust host profile", spec.remediation) + self.assertIn("In NIC mode, do not apply mlxprivhost zero-trust", spec.remediation) + self.assertIn("re-image from a trusted BFB", spec.remediation) + def test_absent_hardware_is_given_no_remediation_to_apply(self) -> None: checks = [ security.SecurityCheck(