diff --git a/README.md b/README.md index e5668ea..c55630d 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,7 @@ audit_profile: - securityVersions.connectxFirmware.status - securityVersions.dcgm.status - securityVersions.dcgmExporter.status + - securityVersions.snap4Container.status - securityVersions.virtioNetBluefield.status - securityVersions.virtioNetBluefield.exposure - securityVersions.dpuHostIsolation.status diff --git a/README_zh.md b/README_zh.md index 3add6ec..981b7d6 100644 --- a/README_zh.md +++ b/README_zh.md @@ -129,6 +129,7 @@ audit_profile: - securityVersions.connectxFirmware.status - securityVersions.dcgm.status - securityVersions.dcgmExporter.status + - securityVersions.snap4Container.status - securityVersions.virtioNetBluefield.status - securityVersions.virtioNetBluefield.exposure - securityVersions.dpuHostIsolation.status diff --git a/cmax/audit_profiles.py b/cmax/audit_profiles.py index 9e28e76..8fc0f4c 100644 --- a/cmax/audit_profiles.py +++ b/cmax/audit_profiles.py @@ -32,6 +32,7 @@ def includes(self, key: str) -> bool: "securityVersions.dcgm.status", "securityVersions.dcgmExporter.status", "securityVersions.virtioNetBluefield.status", + "securityVersions.snap4Container.status", } ), ), diff --git a/cmax/audit_report.py b/cmax/audit_report.py index 8b6244b..04a26c5 100644 --- a/cmax/audit_report.py +++ b/cmax/audit_report.py @@ -112,6 +112,9 @@ def _harness_not_applicable(harness: str) -> str: "securityVersions.virtioNetBluefield.status": ( "The BlueField VIRTIO-Net controller meets the published security minimum." ), + "securityVersions.snap4Container.status": ( + "The BlueField SNAP-4 container meets the published security minimum for its release train." + ), "securityVersions.dpuHostIsolation.status": ( "The host cannot reach the BlueField DPU control plane through the checked interfaces." ), @@ -299,6 +302,7 @@ def _title(key: str) -> str: "Rdma": "RDMA", "Hca": "HCA", "Dpu": "DPU", + "Snap4": "SNAP-4", "Dcgm": "DCGM", "Smmu": "SMMU", "Iommu": "IOMMU", @@ -373,6 +377,11 @@ def _title(key: str) -> str: "custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name," "IMAGES:.spec.containers[*].image' | grep -i dcgm-exporter`" ), + "securityVersions.snap4Container.status": ( + "on each BlueField-3 DPU in DPU mode (locally or over RShim with " + "`ssh ubuntu@192.168.100.2`): `sudo crictl ps` and `sudo crictl images " + "| grep doca_snap`, or `grep -i 'image:.*doca_snap' /etc/kubelet.d/*`" + ), "securityVersions.dpuHostIsolation.status": ( "on every BlueField GPU worker: read `INTERNAL_CPU_RSHIM` with " "`sudo -n mlxconfig -d q`, inspect `/dev/rshim*`, and " diff --git a/cmax/criteria_links.py b/cmax/criteria_links.py index a19b76f..bfeefda 100644 --- a/cmax/criteria_links.py +++ b/cmax/criteria_links.py @@ -93,6 +93,7 @@ "securityVersions.connectxFirmware.status": _SECURITY_PATCHING, "securityVersions.dcgm.status": _SECURITY_PATCHING, "securityVersions.dcgmExporter.status": _SECURITY_PATCHING, + "securityVersions.snap4Container.status": _SECURITY_PATCHING, "securityVersions.virtioNetBluefield.status": _SECURITY_PATCHING, "securityVersions.virtioNetBluefield.exposure": _SECURITY_ISOLATION, "securityVersions.dpuHostIsolation.status": _SECURITY_ISOLATION, diff --git a/cmax/minimum_links.py b/cmax/minimum_links.py index 25b0740..8bef0a6 100644 --- a/cmax/minimum_links.py +++ b/cmax/minimum_links.py @@ -21,6 +21,7 @@ "rocm", "dcgm", "dcgmExporter", + "snap4Container", "ubuntuNoble", } ) @@ -33,6 +34,7 @@ "docker": "docker", "connectx-firmware": "connectxFirmware", "virtio-net-bluefield": "virtioNetBluefield", + "snap4-container": "snap4Container", "fragnesia": "ubuntuNoble", } diff --git a/cmax/scripts/1-audit/audit-common.sh b/cmax/scripts/1-audit/audit-common.sh index 5fa17f5..b3bd12c 100644 --- a/cmax/scripts/1-audit/audit-common.sh +++ b/cmax/scripts/1-audit/audit-common.sh @@ -453,7 +453,7 @@ version_meets_minimum() { # which grades as a warning, instead of "fail", which grades as critical: a # machine proven vulnerable would be softened because a different machine was # unreadable. -VIRTIO_NET_SUMMARY_KEYS="virtioNet virtioNetLine virtioNetMode virtioNetSource virtioNetReason dpuIsolationJson state virtioNetWorstObserved virtioNetWorstObservedLine virtioNetWorstObservedMode virtioNetWorstObservedHost virtioNetObservedJson" +VIRTIO_NET_SUMMARY_KEYS="virtioNet virtioNetLine virtioNetMode virtioNetSource virtioNetReason dpuIsolationJson state virtioNetWorstObserved virtioNetWorstObservedLine virtioNetWorstObservedMode virtioNetWorstObservedHost virtioNetObservedJson snap4Json" # virtio_net_check_facts - resolve the BlueField VIRTIO-Net controller state. # @@ -464,6 +464,8 @@ VIRTIO_NET_SUMMARY_KEYS="virtioNet virtioNetLine virtioNetMode virtioNetSource v # VIRTIO_NET_SOURCE --virtio-net-source (how the version was read) # VIRTIO_NET_REASON --virtio-net-reason (why it was not read) # VIRTIO_NET_ISOLATION_JSON --dpu-isolation-json (the RShim posture evidence) +# SNAP4_JSON --snap4-json (the SNAP-4 container rollup, +# read on the same DPU path) # plus VIRTIO_NET_STATE for the operator log. # # The check self-dispatches per node, because a DPU is per node and a head-node @@ -490,6 +492,7 @@ virtio_net_check_facts() { VIRTIO_NET_WORST_MODE="" VIRTIO_NET_WORST_HOST="" VIRTIO_NET_OBSERVED_JSON="" + SNAP4_JSON="" local check="${WORKLOAD_DIR:-$AUDIT_COMMON_DIR}/checks/fabric/virtio-net-check.py" [[ -f "$check" ]] || check="${AUDIT_COMMON_DIR}/checks/fabric/virtio-net-check.py" @@ -542,6 +545,7 @@ for key in sys.argv[1:]: 9) VIRTIO_NET_WORST_MODE="$line" ;; 10) VIRTIO_NET_WORST_HOST="$line" ;; 11) VIRTIO_NET_OBSERVED_JSON="$line" ;; + 12) SNAP4_JSON="$line" ;; esac index=$((index + 1)) done <<< "$parsed" @@ -688,6 +692,7 @@ build_security_version_audit() { VIRTIO_NET_WORST_HOST="" VIRTIO_NET_OBSERVED_JSON="" VIRTIO_NET_ISOLATION_JSON="" + SNAP4_JSON='{"state":"not_applicable","reason":"scale-out checks do not apply to standalone","observed":[]}' else virtio_net_check_facts fi @@ -713,9 +718,10 @@ build_security_version_audit() { && args+=(--virtio-net-observed-json "$VIRTIO_NET_OBSERVED_JSON") [[ -n "$VIRTIO_NET_ISOLATION_JSON" ]] \ && args+=(--dpu-isolation-json "$VIRTIO_NET_ISOLATION_JSON") + [[ -n "$SNAP4_JSON" ]] && args+=(--snap4-json "$SNAP4_JSON") if [[ ! -f "$policy" ]] || ! command -v python3 >/dev/null 2>&1; then - printf '%s\n' '{"amdDriver":{"status":"unknown"},"nvidiaDriver":{"status":"unknown"},"nvidiaContainerToolkit":{"status":"unknown"},"cudaToolkit":{"status":"unknown"},"docker":{"status":"unknown"},"runc":{"status":"unknown"},"connectxFirmware":{"status":"unknown"},"dcgm":{"status":"unknown"},"dcgmExporter":{"status":"unknown"},"virtioNetBluefield":{"status":"unknown"},"dpuHostIsolation":{"status":"unknown"}}' + printf '%s\n' '{"amdDriver":{"status":"unknown"},"nvidiaDriver":{"status":"unknown"},"nvidiaContainerToolkit":{"status":"unknown"},"cudaToolkit":{"status":"unknown"},"docker":{"status":"unknown"},"runc":{"status":"unknown"},"connectxFirmware":{"status":"unknown"},"dcgm":{"status":"unknown"},"dcgmExporter":{"status":"unknown"},"virtioNetBluefield":{"status":"unknown"},"dpuHostIsolation":{"status":"unknown"},"snap4Container":{"status":"unknown"}}' return 0 fi @@ -760,7 +766,7 @@ build_security_version_audit() { done <<< "$check_output" python3 "$policy" "${args[@]}" 2>/dev/null \ - || printf '%s\n' '{"amdDriver":{"status":"unknown"},"nvidiaDriver":{"status":"unknown"},"nvidiaContainerToolkit":{"status":"unknown"},"cudaToolkit":{"status":"unknown"},"docker":{"status":"unknown"},"runc":{"status":"unknown"},"connectxFirmware":{"status":"unknown"},"dcgm":{"status":"unknown"},"dcgmExporter":{"status":"unknown"},"virtioNetBluefield":{"status":"unknown"},"dpuHostIsolation":{"status":"unknown"}}' + || printf '%s\n' '{"amdDriver":{"status":"unknown"},"nvidiaDriver":{"status":"unknown"},"nvidiaContainerToolkit":{"status":"unknown"},"cudaToolkit":{"status":"unknown"},"docker":{"status":"unknown"},"runc":{"status":"unknown"},"connectxFirmware":{"status":"unknown"},"dcgm":{"status":"unknown"},"dcgmExporter":{"status":"unknown"},"virtioNetBluefield":{"status":"unknown"},"dpuHostIsolation":{"status":"unknown"},"snap4Container":{"status":"unknown"}}' } amd_gpu_check_present() { diff --git a/cmax/scripts/1-audit/audit_findings.py b/cmax/scripts/1-audit/audit_findings.py index c633db0..c23581c 100644 --- a/cmax/scripts/1-audit/audit_findings.py +++ b/cmax/scripts/1-audit/audit_findings.py @@ -507,6 +507,12 @@ def _bluefield_observed(audit: dict[str, Any]) -> bool: ) +def _isolation_nic_mode(audit: dict[str, Any]) -> bool: + """True when every host with an unrestricted RShim read NIC mode.""" + mode = nested_get(audit, "securityVersions", "dpuHostIsolation", "platformMode") + return str(mode or "").strip().lower() == "nic" + + def _bluefield_absence_proven(audit: dict[str, Any]) -> bool: """True only when a completed inventory found no BlueField device.""" isolation = nested_get(audit, "securityVersions", "dpuHostIsolation") @@ -751,6 +757,22 @@ def _tenant_pod_guidance( CONFIG, _status_is("unknown"), ), + # SNAP-4 runs on the DPU Arm side, so it is graded only where the DPU is + # part of the audited fabric, the same scope as the virtio-net controller. + Rule( + "securityVersions.snap4Container.status", + "BlueField SNAP-4 container below security minimum", + VERSION, + _status_is("fail"), + harnesses=SCALE_OUT_HARNESSES, + ), + Rule( + "securityVersions.snap4Container.status", + "BlueField SNAP-4 container version requires provider attestation", + CONFIG, + _status_is("unknown"), + harnesses=SCALE_OUT_HARNESSES, + ), # --- BlueField DPU ---------------------------------------------------- # The controller firmware check has four reportable states, not two. A # below-minimum controller in DPU mode is running and fails. The same firmware @@ -857,6 +879,18 @@ def _tenant_pod_guidance( "--disable_counter_rd --disable_port_owner", CONFIG, _status_is("fail"), + guard=lambda audit: not _isolation_nic_mode(audit), + harnesses=SCALE_OUT_HARNESSES, + ), + Rule( + "securityVersions.dpuHostIsolation.status", + "Host side can open RShim to the BlueField (NIC mode, " + "INTERNAL_CPU_RSHIM=0); keep tenant host root off the card or " + "re-image it per tenant, not mlxprivhost, which NVIDIA does not " + "support in NIC mode", + CONFIG, + _status_is("fail"), + guard=_isolation_nic_mode, harnesses=SCALE_OUT_HARNESSES, ), Rule( @@ -1640,6 +1674,8 @@ def classify_suppression(key: str, audit: dict[str, Any]) -> tuple[str, str]: "securityVersions.connectxFirmware.status": "security-connectx-firmware", "securityVersions.dcgm.status": None, "securityVersions.dcgmExporter.status": None, + # CLI-only until the criteria catalog has a SNAP-4 criterion. + "securityVersions.snap4Container.status": None, "securityVersions.virtioNetBluefield.status": "security-virtio-net-bluefield", "securityVersions.virtioNetBluefield.exposure": "security-virtio-net-bluefield", "securityVersions.dpuHostIsolation.status": "security-dpu-host-isolation", @@ -1738,6 +1774,7 @@ def _audit_data(values: dict[str, Any]) -> dict[str, Any]: "nvidiaContainerToolkit": "NVIDIA Container Toolkit", "nvidiaDriver": "NVIDIA driver", "runc": "runc", + "snap4Container": "BlueField SNAP-4 container", "virtioNetBluefield": "BlueField VIRTIO-Net controller", } @@ -1915,11 +1952,31 @@ def _finding_guidance( "mlxprivhost -d r --disable_rshim --disable_tracer " "--disable_counter_rd --disable_port_owner" ) - recommendation = ( - "Apply zero-trust mode on every affected BlueField with " - f"`{remediation}`. Then verify that `INTERNAL_CPU_RSHIM=1`, " - "`/dev/rshim0` is absent, and `tmfifo_net0` is absent." + nic_mode = ( + isinstance(isolation, dict) + and str(isolation.get("platformMode") or "").strip().lower() == "nic" ) + if nic_mode: + # NVIDIA does not support the zero-trust profile in NIC mode. + recommendation = ( + "Two ways to make this real isolation, not just a setting a " + "tenant can undo. Either keep tenant host root off the " + "BlueField (sudo -M, mdev-coldplug, or an IOMMU/ACS isolation " + "of the PCI function), so only the provider can reach it. Or " + "accept that a tenant can own the card and re-image it from a " + "known-good BFB before each new tenant and after each one " + "leaves. `mlxconfig -d s INTERNAL_CPU_RSHIM=1` plus a " + "power cycle is real only under the first path; NVIDIA does not " + "support the zero-trust mlxprivhost profile in NIC mode. Then " + "verify that `INTERNAL_CPU_RSHIM=1`, `/dev/rshim0` is absent, " + "and `tmfifo_net0` is absent." + ) + else: + recommendation = ( + "Apply zero-trust mode on every affected BlueField with " + f"`{remediation}`. Then verify that `INTERNAL_CPU_RSHIM=1`, " + "`/dev/rshim0` is absent, and `tmfifo_net0` is absent." + ) elif ( rule.key == "securityVersions.dpuHostIsolation.status" and str(value).strip().lower() == "unknown" diff --git a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py index 5f12b79..30da76d 100755 --- a/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py +++ b/cmax/scripts/1-audit/checks/fabric/virtio-net-check.py @@ -148,6 +148,9 @@ # Per-command timeout for every local tool. SLURM_TIMEOUT_S below is derived # from it, so the two move together. TOOL_TIMEOUT_S = 15 +# The full mlxconfig dump, read only when firmware rejects a named parameter. +# It took 22 s idle and about 60 s under contention on a GB300 BlueField-3. +MLXCONFIG_FULL_TIMEOUT_S = 60 # ssh gets ConnectTimeout=SSH_TIMEOUT_S and a wall-clock budget of twice that, # so a connection that opens and then stalls is still bounded. SSH_TIMEOUT_S = 8 @@ -159,19 +162,21 @@ # Worst case for one host, every command hanging until its own timeout: # # lspci 1 x 15 = 15 s -# mlxconfig q, sudo -n mlxconfig q, -# sudo -n mst start, sudo -n mlxconfig q 4 x 15 = 60 s +# named mlxconfig q, sudo -n named q 2 x 15 = 30 s +# full mlxconfig q, only after a named read +# answered "Unknown Parameter" 1 x 60 = 60 s # local `virtnet version` + `virtnet -v` 2 x 15 = 30 s # RShim SSH: version, -v, list, mlnx-release 4 x 16 = 64 s +# SNAP-4 listing, one RShim SSH command 1 x 16 = 16 s # ------ -# 169 s +# 215 s # -# The local-virtnet-succeeded path and the RShim SSH path are mutually -# exclusive (collect_version only reaches SSH when the local read produced no -# version), and the succeeded path is cheaper: 15 + 60 + 15 + 2 x 15 = 120 s. -# So 169 s is a true ceiling, not a sum of alternatives. Allow 60 s for srun -# dispatch and worker startup, giving 229 s against a 300 s budget, about 24% -# margin. +# The full dump runs only after one of the named reads answered, so the mode +# read is at most 15 + 15 + 60 = 90 s. The local-virtnet-succeeded path and the +# RShim SSH path are mutually exclusive (collect_version only reaches SSH when +# the local read produced no version), and the succeeded path is cheaper. So +# 215 s is a true ceiling, not a sum of alternatives. Allow 60 s for srun +# dispatch and worker startup, giving 275 s against a 300 s budget. # # Redo this arithmetic when changing TOOL_TIMEOUT_S or SSH_TIMEOUT_S, or when # adding a command to gather(). An overrun is no longer a false clean, because @@ -475,10 +480,49 @@ def pci_scan_result( } +def sysfs_pci_listing(env: dict[str, str] = os.environ) -> tuple[str, str | None]: + """Synthesize an `lspci -Dn` style listing from ``/sys/bus/pci/devices``. + + Container images used as compute nodes (Slurm-on-Kubernetes slurmd pods, + for example) often ship without pciutils while the host's sysfs is still + mounted. The kernel's own vendor/device/class files answer the same + question lspci would, so they are read when lspci cannot run. Any entry + whose IDs cannot be read makes the listing partial: the devices that were + read still count as positive evidence, but absence is not claimed. + """ + root = Path(env.get("CLUSTERMAX_AUDIT_ROOT", "") + "/sys/bus/pci/devices") + try: + entries = sorted(root.iterdir()) + except OSError as exc: + return "", f"sysfs PCI tree unreadable: {exc.strerror or exc}" + lines: list[str] = [] + unreadable = 0 + for entry in entries: + try: + vendor = (entry / "vendor").read_text().strip().lower().removeprefix("0x") + device = (entry / "device").read_text().strip().lower().removeprefix("0x") + pci_class = (entry / "class").read_text().strip().lower().removeprefix("0x") + except OSError: + unreadable += 1 + continue + lines.append(f"{entry.name} {pci_class[:4]}: {vendor}:{device}") + if unreadable: + return "\n".join(lines), f"sysfs could not read {unreadable} PCI device(s)" + return "\n".join(lines), None + + def collect_pci( *, runner: Runner = subprocess.run, env: dict[str, str] = os.environ ) -> dict[str, Any]: text, error = run_tool(["lspci", "-Dnn"], runner=runner) + if error is not None: + sys_text, sys_error = sysfs_pci_listing(env) + if sys_text: + result = pci_scan_result(sys_text, sys_error) + result["pciSource"] = "sysfs" + if not result["scanComplete"]: + result["scanError"] = f"{error}; {sys_error}" + return result return pci_scan_result( text, error, empty_bus_confirmed=sysfs_pci_bus_empty(env) ) @@ -531,6 +575,30 @@ def parse_mlxconfig(text: str) -> dict[str, str]: return values +def mlxconfig_query(slot: str, *, named: bool = True) -> list[str]: + """The mode read: only the parameters the check grades. + + The full `mlxconfig -e -d q` dump walks every parameter on the card. + On VP's GB300 BlueField-3 hosts it took 22 s on an idle host and about 60 s + with three reads in flight, past TOOL_TIMEOUT_S, so 14 of 16 hosts timed out + and graded the mode unreadable. Naming the three parameters answers in under + 2 s. ``named=False`` is the fallback for firmware that rejects a name. + """ + query = ["mlxconfig", "-e", "-d", slot, "q"] + return [*query, *_MLXCONFIG_KEYS] if named else query + + +def _unknown_parameter(text: str) -> bool: + """mlxconfig rejected a named parameter, so the device did open. + + MFT words this two ways: ``-E- Unknown Parameter: X`` for a name it does not + know, and ``-E- The Device doesn't support X parameter`` (seen on a + ConnectX-8 with MFT 4.33) for one the firmware lacks. + """ + text = text or "" + return "Unknown Parameter" in text or "doesn't support" in text + + def privileged_prefix( *, which: Which = shutil.which, @@ -595,18 +663,34 @@ def collect_mode( return result slot = devices[0].get("slot") or "" - query = ["mlxconfig", "-e", "-d", slot, "q"] + query = mlxconfig_query(slot) prefix, escalation = privileged_prefix(which=which, euid=euid) result["escalation"] = escalation source = "mlxconfig" text, error = run_tool(query, runner=runner) values = parse_mlxconfig(text) - if not values and prefix: + answered: list[str] | None = [] if values or _unknown_parameter(text) else None + if not values and answered is None and prefix: text, sudo_error = run_tool([*prefix, *query], runner=runner) values = parse_mlxconfig(text) error = error or sudo_error + if values or _unknown_parameter(text): + answered = prefix if values: source = "sudo mlxconfig" + if not values and answered is not None and _unknown_parameter(text): + # Firmware that predates one of the named parameters rejects the whole + # named query. Only then is the full dump read, once, under the + # privilege that already opened the device, with its own longer budget. + text, full_error = run_tool( + [*answered, *mlxconfig_query(slot, named=False)], + runner=runner, + timeout=MLXCONFIG_FULL_TIMEOUT_S, + ) + values = parse_mlxconfig(text) + error = full_error or error + if values: + source = "sudo mlxconfig" if answered else "mlxconfig" if not values: result["error"] = error or "mlxconfig returned no readable Current mode parameters" return result @@ -813,6 +897,31 @@ def collect_line( return line, evidence +def dpu_ssh_prefix(env: dict[str, str] = os.environ) -> list[str]: + """The RShim SSH argv every DPU-side read starts with. + + BatchMode=yes cannot prompt for a password or a passphrase, so this either + works with an already-trusted key or fails immediately. Nothing is ever + written to the RShim console and no interactive session is opened. + UserKnownHostsFile=/dev/null keeps the read-only audit from leaving a host + key behind in the operator's known_hosts, which StrictHostKeyChecking=no + would otherwise write on the first run. + """ + target = env.get("CLUSTERMAX_DPU_SSH_TARGET") or "ubuntu@192.168.100.2" + return [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + f"ConnectTimeout={SSH_TIMEOUT_S}", + target, + ] + + def collect_version( rshim: dict[str, Any], *, @@ -849,26 +958,7 @@ def empty(cli: str | None) -> dict[str, Any]: if not (rshim.get("rshimDeviceNode") or rshim.get("tmfifoNet0")): return empty(local_cli) - target = env.get("CLUSTERMAX_DPU_SSH_TARGET") or "ubuntu@192.168.100.2" - # BatchMode=yes cannot prompt for a password or a passphrase, so this either - # works with an already-trusted key or fails immediately. Nothing is ever - # written to the RShim console and no interactive session is opened. - # UserKnownHostsFile=/dev/null keeps the read-only audit from leaving a host - # key behind in the operator's known_hosts, which StrictHostKeyChecking=no - # would otherwise write on the first run. - ssh = [ - "ssh", - "-o", - "BatchMode=yes", - "-o", - "StrictHostKeyChecking=no", - "-o", - "UserKnownHostsFile=/dev/null", - "-o", - f"ConnectTimeout={SSH_TIMEOUT_S}", - target, - "virtnet", - ] + ssh = [*dpu_ssh_prefix(env), "virtnet"] remote = read_virtnet(ssh, runner=runner, timeout=SSH_TIMEOUT_S * 2) if remote is None: rshim["dpuReachedFromHost"] = False @@ -887,6 +977,298 @@ def empty(cli: str | None) -> dict[str, Any]: } +# --------------------------------------------------------------------------- +# SNAP-4 container (NVIDIA bulletin a_id 5744) +# --------------------------------------------------------------------------- +# +# DOCA SNAP-4 runs as a container on the BlueField-3 Arm cores, started by the +# DPU's kubelet from a static pod manifest in /etc/kubelet.d. Its version is the +# image tag (nvcr.io/nvidia/doca/doca_snap:4.9.1-doca3.2.0). Nothing on the host +# side carries it, so it is read on the same DPU access path as the virtio-net +# controller: locally when this check already runs on the DPU, otherwise over +# the RShim SSH link. One remote command does the whole read, so an unreachable +# DPU costs one SSH timeout. +# +# The read only lists: it never pulls an image, starts or stops a container, or +# execs into one. `crictl` needs root on a stock DPU image, so it is retried +# under `sudo -n`, which never prompts. + +SNAP4_STATE_NOT_INSTALLED = "not_installed" + +# Running containers first, then the kubelet manifests. kubelet keeps a static +# pod running for as long as its manifest exists, so a manifest is the version +# that runs whenever the container is restarted. +SNAP4_DPU_SCRIPT = r""" +echo @@SNAP4-MANIFESTS@@ +for f in /etc/kubelet.d/*; do + [ -f "$f" ] && [ -r "$f" ] && grep -i 'image:.*doca_snap' "$f" 2>/dev/null +done +if ! command -v crictl >/dev/null 2>&1; then + echo @@SNAP4-NO-CRICTL@@ + exit 0 +fi +echo @@SNAP4-PS@@ +crictl ps -o json 2>/dev/null || sudo -n crictl ps -o json 2>/dev/null || echo @@SNAP4-PS-FAILED@@ +# The bare echoes keep a marker on its own line when the JSON has no trailing +# newline. +echo +echo @@SNAP4-IMAGES@@ +crictl images -o json 2>/dev/null || sudo -n crictl images -o json 2>/dev/null +echo +echo @@SNAP4-END@@ +""" + +_SNAP4_IMAGE_RE = re.compile(r"doca_snap[^\s:@\"']*:v?(\d+\.\d+\.\d+)", re.IGNORECASE) + + +def snap4_version_from_image(image: str) -> str | None: + match = _SNAP4_IMAGE_RE.search(image or "") + return match.group(1) if match else None + + +def _snap4_sections(text: str) -> dict[str, str]: + sections: dict[str, list[str]] = {} + current: str | None = None + for line in (text or "").splitlines(): + marker = line.strip() + if marker.startswith("@@SNAP4-") and marker.endswith("@@"): + current = marker[len("@@SNAP4-") : -2] + sections.setdefault(current, []) + continue + if current is not None: + sections[current].append(line) + return {name: "\n".join(lines) for name, lines in sections.items()} + + +def _json_or_none(text: str) -> Any: + try: + return json.loads(text) + except (json.JSONDecodeError, TypeError): + return None + + +def parse_snap4_listing(text: str) -> dict[str, Any]: + """Read the SNAP4_DPU_SCRIPT output. + + ``reached`` says the script ran on the DPU at all. ``listed`` says crictl + answered, which is what makes "no SNAP-4 container here" a finding rather + than a guess. ``running`` and ``configured`` are the versions read off the + running containers and the kubelet manifests. + """ + sections = _snap4_sections(text) + result: dict[str, Any] = { + "reached": "MANIFESTS" in sections, + "listed": False, + "running": [], + "runningWithoutVersion": False, + "configured": [], + "error": None, + } + if not result["reached"]: + result["error"] = "the DPU-side read produced no output" + return result + for line in sections.get("MANIFESTS", "").splitlines(): + version = snap4_version_from_image(line) + if version and version not in result["configured"]: + result["configured"].append(version) + if "NO-CRICTL" in sections: + result["error"] = "crictl is not installed on the DPU" + return result + ps_text = sections.get("PS", "") + if "PS-FAILED" in sections: + result["error"] = "crictl ps did not answer, even under sudo -n" + return result + containers = (_json_or_none(ps_text) or {}).get("containers") + if not isinstance(containers, list): + result["error"] = "crictl ps returned no readable container list" + return result + result["listed"] = True + tags: dict[str, list[str]] = {} + images = (_json_or_none(sections.get("IMAGES", "")) or {}).get("images") or [] + for image in images if isinstance(images, list) else []: + if not isinstance(image, dict): + continue + names = [str(tag) for tag in image.get("repoTags") or []] + for key in (image.get("id"), *(image.get("repoDigests") or [])): + if key: + tags[str(key)] = names + for container in containers: + if not isinstance(container, dict): + continue + image_field = container.get("image") + image_name = str(image_field.get("image") if isinstance(image_field, dict) else image_field or "") + names = [image_name, *tags.get(image_name, []), *tags.get(str(container.get("imageRef") or ""), [])] + container_name = str((container.get("metadata") or {}).get("name") or "") + is_snap = any("doca_snap" in name.lower() for name in names) or "snap" in container_name.lower() + if not is_snap: + continue + versions = [version for version in map(snap4_version_from_image, names) if version] + if versions: + if versions[0] not in result["running"]: + result["running"].append(versions[0]) + else: + result["runningWithoutVersion"] = True + return result + + +def collect_snap4( + rshim: dict[str, Any], + *, + on_dpu: bool, + env: dict[str, str] = os.environ, + runner: Runner = subprocess.run, +) -> dict[str, Any]: + """Read the SNAP-4 container version on the DPU, if this host can reach it.""" + if on_dpu: + command, via, timeout = ["sh", "-c", SNAP4_DPU_SCRIPT], "local", TOOL_TIMEOUT_S + elif env.get("CLUSTERMAX_DPU_SSH_DISABLE"): + return {"attempted": False, "via": None, "reason": "RShim SSH is disabled"} + elif not (rshim.get("rshimDeviceNode") or rshim.get("tmfifoNet0")): + return {"attempted": False, "via": None, "reason": "no RShim link to the DPU"} + else: + command, via, timeout = [*dpu_ssh_prefix(env), SNAP4_DPU_SCRIPT], "rshim-ssh", SSH_TIMEOUT_S * 2 + text, error = run_tool(command, runner=runner, timeout=timeout) + listing = parse_snap4_listing(text) + listing["attempted"] = True + listing["via"] = via + if not listing["reached"]: + listing["error"] = error or listing["error"] + elif via == "rshim-ssh": + # The script ran on the DPU, so the host did reach it. + rshim["dpuReachedFromHost"] = True + return listing + + +def decide_snap4(record: dict[str, Any], read: dict[str, Any]) -> dict[str, Any]: + """Per-host SNAP-4 state, from the host record and the DPU-side read.""" + via = read.get("via") + if read.get("running"): + return { + "state": STATE_VERSION, + "versions": list(read["running"]), + "source": f"{via}/crictl", + "reason": "SNAP-4 image tag read from the running container", + } + if read.get("configured"): + return { + "state": STATE_VERSION, + "versions": list(read["configured"]), + "source": f"{via}/kubelet-manifest", + "reason": ( + "SNAP-4 image tag read from the kubelet manifest; no running " + "container was listed, and kubelet starts this version on restart" + ), + } + base = {"versions": [], "source": None} + if read.get("runningWithoutVersion"): + return { + **base, + "state": STATE_UNKNOWN, + "reason": "a SNAP-4 container runs on the DPU but its image reference carries no version tag", + } + if not record.get("scanComplete"): + return {**base, "state": STATE_INCOMPLETE, "reason": str(record.get("scanError") or "PCI scan could not run")} + if not record.get("bluefield3Present"): + return {**base, "state": STATE_NOT_APPLICABLE, "reason": "no BlueField-3 device is present on this host"} + if record.get("mode") == "nic": + return { + **base, + "state": STATE_NOT_RUNNING, + "reason": ( + "BlueField-3 is in NIC mode (INTERNAL_CPU_OFFLOAD_ENGINE=1), so the DPU " + "Arm cores that run SNAP-4 are off. A change back to DPU mode would " + "start any installed SNAP-4 container unchanged." + ), + } + if read.get("listed"): + return { + **base, + "state": SNAP4_STATE_NOT_INSTALLED, + "source": f"{via}/crictl", + "reason": "the DPU answered and runs no SNAP-4 container, and no kubelet manifest names one", + } + if record.get("mode") != "dpu": + return { + **base, + "state": STATE_INCOMPLETE, + "reason": "BlueField-3 is present but its mode could not be read, so SNAP-4 scope is unsettled", + } + if read.get("reached"): + detail = f"the DPU answered but {read.get('error') or 'listed nothing'}" + elif read.get("attempted"): + detail = f"the DPU did not answer over RShim ({read.get('error') or 'no output'})" + else: + detail = str(read.get("reason") or "the DPU was not reached") + return { + **base, + "state": STATE_UNKNOWN, + "reason": ( + "BlueField-3 is in DPU mode and the SNAP-4 container is observable only " + f"on the DPU Arm side; {detail}. The provider must attest the SNAP-4 " + "version or confirm it is not deployed." + ), + } + + +# Most severe first. not_running ranks above version for the same reason as on +# the controller ladder: a NIC-mode host was not read, so a read peer must not +# speak for it in the reported state. The evaluator grades the readings +# themselves from `observed`, so this order never softens a fail. +SNAP4_LADDER = ( + STATE_UNKNOWN, + STATE_INCOMPLETE, + STATE_NOT_RUNNING, + STATE_VERSION, + SNAP4_STATE_NOT_INSTALLED, + STATE_NOT_APPLICABLE, +) +SNAP4_GAP_RUNG = STATE_INCOMPLETE + + +def _snap4_rung(record: dict[str, Any]) -> str: + snap = record.get("snap4") if isinstance(record.get("snap4"), dict) else {} + state = str(snap.get("state") or STATE_INCOMPLETE) + return state if state in SNAP4_LADDER else STATE_INCOMPLETE + + +def summarize_snap4(records: list[dict[str, Any]]) -> dict[str, Any]: + """Roll the per-host SNAP-4 states into the cluster answer. + + ``observed`` carries every distinct version a host read, whatever the + rollup state, so a below-minimum DPU stays a finding beside an unread peer. + SNAP-4 is fixed per release train, so the readings are not ordered here; the + evaluator grades each one against its own train. + """ + if not records: + return { + "state": STATE_INCOMPLETE, + "reason": "no host produced a SNAP-4 record", + "observed": [], + "hostsChecked": 0, + } + state = _snap4_rung(most_severe(records, ladder=SNAP4_LADDER, rung_of=_snap4_rung)) + seen: dict[tuple[str, str], dict[str, Any]] = {} + for record in records: + snap = record.get("snap4") if isinstance(record.get("snap4"), dict) else {} + for version in snap.get("versions") or []: + key = (str(version), str(snap.get("source") or "")) + seen.setdefault(key, {"version": str(version), "host": record.get("host"), "source": snap.get("source")}) + observed = sorted(seen.values(), key=lambda entry: (_version_key(entry["version"]), str(entry["host"] or ""))) + reasons = sorted( + { + str((record.get("snap4") or {}).get("reason") or "") + for record in records + if _snap4_rung(record) == state + } + ) + return { + "state": state, + "reason": "; ".join(reason for reason in reasons if reason), + "observed": observed, + "hostsChecked": len(records), + } + + # --------------------------------------------------------------------------- # State machine # --------------------------------------------------------------------------- @@ -927,6 +1309,10 @@ def decide(record: dict[str, Any]) -> tuple[str, str]: ), ESCALATION_ROOT: "mlxconfig ran with root rights and still reported no mode", }.get(str(record.get("modeEscalation")), "mlxconfig needs root") + if "is not installed" in str(record.get("modeError") or ""): + # A missing tool is not a privilege problem. Saying sudo was + # refused here would send an operator after the wrong fix. + privilege = "install NVIDIA MFT, which provides mlxconfig, to read it" return ( STATE_INCOMPLETE, "BlueField-3 is present but its mode could not be read " @@ -968,6 +1354,7 @@ def gather( "host": env.get("NODE_NAME") or socket.gethostname(), "scanComplete": pci["scanComplete"], "scanError": pci["scanError"], + "pciSource": pci.get("pciSource", "lspci"), "bluefield3Present": bool(devices) if pci["scanComplete"] else None, "bluefieldDevices": devices, "virtioPciDevices": pci["virtioPciDevices"], @@ -985,6 +1372,13 @@ def gather( "lineEvidence": version.get("lineEvidence") or [], } record["state"], record["reason"] = decide(record) + on_dpu = bool(version.get("versionSource")) and not str(version["versionSource"]).startswith("rshim-ssh/") + if record["bluefield3Present"] and record["mode"] != "nic": + snap_read = collect_snap4(rshim, on_dpu=on_dpu, env=env, runner=runner) + else: + # No DPU, or its Arm cores are off: there is nothing to reach. + snap_read = {"attempted": False, "reason": "no DPU Arm side to read"} + record["snap4"] = decide_snap4(record, snap_read) return record @@ -1117,6 +1511,7 @@ def summarize(records: list[dict[str, Any]]) -> dict[str, Any]: "bluefield3Present": None, "rshimHostAccess": {}, }, + "snap4": summarize_snap4(records), } observed = observed_controllers(records) @@ -1170,6 +1565,7 @@ def summarize(records: list[dict[str, Any]]) -> dict[str, Any]: "observedControllers": observed, "worstObserved": worst_observed(observed), "isolationEvidence": isolation_evidence(records), + "snap4": summarize_snap4(records), } @@ -1229,6 +1625,16 @@ def apply_coverage_gap(summary: dict[str, Any], errors: list[str]) -> dict[str, evidence["scanComplete"] = False evidence["scanError"] = f"the per-node scan did not reach every host ({gap})" summary["isolationEvidence"] = evidence + + # SNAP-4 is per DPU too, so the same gap withdraws its clean claims. The + # readings in `observed` stay, so a below-minimum DPU is still a finding. + snap4 = dict(summary.get("snap4") or {}) + if snap4 and gap_can_weaken(snap4.get("state"), ladder=SNAP4_LADDER, gap_rung=SNAP4_GAP_RUNG): + snap4["state"] = SNAP4_GAP_RUNG + prior_snap = str(snap4.get("reason") or "").strip().rstrip(";") + snap_note = f"the per-node scan did not reach every host ({gap})" + snap4["reason"] = f"{prior_snap}; {snap_note}" if prior_snap else snap_note + summary["snap4"] = snap4 return summary @@ -1323,6 +1729,19 @@ def _isolation_proof_rung(record: dict[str, Any]) -> str: return ISO_REACHABLE if _isolation_reachable(record) else ISO_POSTURE_GAP +def _isolation_platform_mode(records: list[dict[str, Any]]) -> str: + unrestricted = [ + record + for record in records + if (record.get("rshimHostAccess") or {}).get("rshimRestricted") is False + or _isolation_path_reachable(record) + ] + if not unrestricted: + return "unknown" + modes = {str(record.get("mode") or "unknown").lower() for record in unrestricted} + return "nic" if modes == {"nic"} else ("dpu" if "dpu" in modes else "unknown") + + def isolation_evidence(records: list[dict[str, Any]]) -> dict[str, Any]: """The one host record the DPU host-isolation verdict is graded from. @@ -1345,6 +1764,12 @@ def isolation_evidence(records: list[dict[str, Any]]) -> dict[str, Any]: "bluefield3Present": chosen.get("bluefield3Present"), "modeError": chosen.get("modeError"), "rshimHostAccess": chosen.get("rshimHostAccess") or {}, + # NIC mode is host-trusted by design and NVIDIA does not support the + # zero-trust (mlxprivhost restricted) profile there, so the verdict + # grades an unrestricted RShim differently in NIC mode. It is "nic" only + # when every host whose RShim is unrestricted read NIC mode; one host in + # DPU mode, or one whose mode is unread, keeps the DPU-mode grade. + "platformMode": _isolation_platform_mode(records), "unassessedHosts": sorted( str(record.get("host") or "") for record in records @@ -1466,7 +1891,11 @@ def slurm_records( printf '@@MODE_ERROR@@\t%s\n' "Multiple BlueField-3 PCI devices require per-device mode evidence" elif [ -n "$slot" ] && command -v mlxconfig >/dev/null 2>&1; then echo @@MLXCONFIG_BEGIN@@ - mlxconfig -e -d "$slot" q 2>&1 + mode_out=$(mlxconfig -e -d "$slot" q INTERNAL_CPU_MODEL INTERNAL_CPU_OFFLOAD_ENGINE INTERNAL_CPU_RSHIM 2>&1) + case "$mode_out" in + *"Unknown Parameter"*|*"support INTERNAL_CPU"*) mode_out=$(mlxconfig -e -d "$slot" q 2>&1) ;; + esac + printf '%s\n' "$mode_out" echo @@MLXCONFIG_END@@ elif [ -n "$slot" ]; then printf '@@MODE_ERROR@@\t%s\n' "mlxconfig is not installed" @@ -1572,6 +2001,11 @@ def record_from_k8s(node: str, parsed: dict[str, Any]) -> dict[str, Any]: "lineEvidence": [parsed["virtnet"]] if version else [], } record["state"], record["reason"] = decide(record) + # The driver pod reads the host, not the DPU, so SNAP-4 stays unread here. + record["snap4"] = decide_snap4( + record, + {"attempted": False, "reason": "the Kubernetes driver-pod read does not reach the DPU"}, + ) return record @@ -1782,6 +2216,7 @@ def collector_summary(payload: dict[str, Any]) -> dict[str, Any]: "virtioNetWorstObservedHost": worst.get("host") or "", "virtioNetObservedJson": json.dumps(observed, sort_keys=True), "dpuIsolationJson": json.dumps(summary.get("isolationEvidence") or {}, sort_keys=True), + "snap4Json": json.dumps(summary.get("snap4") or {}, sort_keys=True), "state": state, "reason": summary.get("reason") or "", } diff --git a/cmax/scripts/1-audit/security_fleet.py b/cmax/scripts/1-audit/security_fleet.py index 8a4d8e1..991f386 100644 --- a/cmax/scripts/1-audit/security_fleet.py +++ b/cmax/scripts/1-audit/security_fleet.py @@ -29,9 +29,9 @@ from typing import Any # Checks whose evidence lives on each GPU host. Cluster-level checks (the -# dcgm-exporter image, the CUDA runtime pods, and the BlueField VIRTIO-Net and -# DPU host-isolation results, which both come from the fabric-wide virtio-net -# check) already have their own scope and are left as the collector built them. +# dcgm-exporter image, the CUDA runtime pods, and the BlueField VIRTIO-Net, +# SNAP-4, and DPU host-isolation results, which all come from the fabric-wide +# virtio-net check) already have their own scope and are left as the collector built them. VERSION_KEYS = ( "nvidiaDriver", "amdDriver", diff --git a/cmax/scripts/1-audit/security_version_audit.py b/cmax/scripts/1-audit/security_version_audit.py index f08b05b..6f5f8fe 100644 --- a/cmax/scripts/1-audit/security_version_audit.py +++ b/cmax/scripts/1-audit/security_version_audit.py @@ -437,6 +437,199 @@ def dcgm_exporter_verdict(version: str) -> Verdict: return _minimum_kind_verdict(version, "dcgmExporter", label="DCGM Exporter") +# --------------------------------------------------------------------------- +# SNAP-4 container (NVIDIA bulletin a_id 5744) +# --------------------------------------------------------------------------- + +SNAP4_COMPONENT = "snap4Container" +# Rollup states where every host was either read or cannot run SNAP-4 now, so a +# set of passing readings speaks for the cluster. +SNAP4_COMPLETE_STATES = frozenset({"version", "not_installed", "not_running", "not_applicable"}) + + +def _snap4_trains() -> tuple[dict[str, str], str, dict[str, Any]]: + """(train -> fixed version, minimum text, component block). Raises on a missing table.""" + block = minimum_versions.component(SNAP4_COMPONENT) + lines = block.get("lines") if isinstance(block.get("lines"), dict) else {} + fixed = { + str(train): str(entry["fixed"]) + for train, entry in lines.items() + if isinstance(entry, dict) and numeric_version(str(entry.get("fixed") or "")) is not None + } + ordered = sorted(fixed.items(), key=lambda item: numeric_version(item[1]) or ()) + text = ", ".join(f"{version} ({train} train)" for train, version in ordered) + return fixed, text or MINIMUMS_UNAVAILABLE, block + + +def _snap4_minimum_text() -> str: + try: + return _snap4_trains()[1] + except MinimumDataError: + return MINIMUMS_UNAVAILABLE + + +def snap4_verdict(version: str) -> tuple[Verdict, str | None]: + """Grade one SNAP-4 version against the fixed version of its own train. + + NVIDIA fixes SNAP-4 per major.minor train ("SNAP-4.9.1, SNAP4 4.5.5"), and + the bulletin marks everything below the newest fix as affected except the + fixed trains themselves. So a version on a listed train is held to that + train's fix, a version newer than every fix carries them all, and a version + between or below the trains has no fix of its own and must move to the next + fixed release above it. Returns the verdict and the train its minimum came + from, which selects the grace window. + """ + try: + fixed, minimum_text, block = _snap4_trains() + except MinimumDataError as exc: + return _minimums_unavailable(version, exc), None + advisory = _advisory(block) + cited = _cve_detail(block, "SNAP-4 security bulletin minimum") + if not fixed: + return ( + _unknown(version, MINIMUMS_UNAVAILABLE, advisory, "snap4Container lists no fixed release train"), + None, + ) + observed = numeric_version(version, parts=3) + if observed is None: + return _unknown(version, minimum_text, advisory, f"SNAP-4 version {version!r} is unreadable"), None + train = f"{observed[0]}.{observed[1]}" + if train in fixed: + minimum = fixed[train] + passed = observed >= (numeric_version(minimum, parts=3) or ()) + detail = ( + f"Meets the {cited} for the SNAP-4 {train} train ({minimum})" + if passed + else f"Below the {cited} for the SNAP-4 {train} train ({minimum})" + ) + return Verdict(version, "pass" if passed else "fail", minimum, advisory, detail), train + ordered = sorted(fixed.items(), key=lambda item: numeric_version(item[1], parts=3) or ()) + highest_train, highest = ordered[-1] + if observed > (numeric_version(highest, parts=3) or ()): + return ( + Verdict( + version, + "pass", + highest, + advisory, + f"Newer than every fixed SNAP-4 train in the {cited} (newest fix {highest}), " + "so it carries those fixes", + ), + highest_train, + ) + above = next( + (item for item in ordered if (numeric_version(item[1], parts=3) or ()) > observed), + ordered[-1], + ) + return ( + Verdict( + version, + "fail", + above[1], + advisory, + f"Below the {cited}: the SNAP-4 {train} train has no fixed release, so the " + f"nearest fix is {above[1]} ({above[0]} train)", + ), + above[0], + ) + + +def snap4_container_result(summary: dict[str, Any] | None) -> dict[str, object]: + """Grade the SNAP-4 rollup the virtio-net check produced. + + Every reading is graded on its own train and the worst one decides. A + below-minimum reading fails whatever the coverage, and a passing reading + clears the cluster only when every host was read or cannot run SNAP-4 now. + """ + advisory = _component_advisory(SNAP4_COMPONENT) + minimum_text = _snap4_minimum_text() + if not isinstance(summary, dict) or not summary: + return asdict( + _unknown( + "unknown", + minimum_text, + advisory, + "The SNAP-4 container read did not run, so its version was not observed", + ) + ) + state = str(summary.get("state") or "incomplete") + reason = str(summary.get("reason") or "").strip() + observed = [entry for entry in summary.get("observed") or [] if isinstance(entry, dict) and entry.get("version")] + complete = state in SNAP4_COMPLETE_STATES + + def finish(verdict: Verdict, selector: str | None, entry: dict[str, Any] | None) -> dict[str, object]: + record = _verdict_record(verdict, SNAP4_COMPONENT if selector else None, selector) + record["state"] = state + if entry: + record["host"] = entry.get("host") + record["source"] = entry.get("source") + if observed: + record["observed"] = observed + return record + + graded = [(entry, *snap4_verdict(str(entry["version"]))) for entry in observed] + for status in ("fail", "unknown"): + hits = [item for item in graded if item[1].status == status] + if not hits: + continue + entry, verdict, selector = min(hits, key=lambda item: numeric_version(item[1].version, parts=3) or ()) + detail = f"{verdict.detail}, read on {entry.get('host') or 'a host'} via {entry.get('source') or 'unknown'}" + if not complete and status == "fail": + detail += f". Not every DPU was read ({reason or state})" + return finish(replace(verdict, detail=detail), selector if status == "fail" else None, entry) + if graded: + entry, verdict, selector = min(graded, key=lambda item: numeric_version(item[1].version, parts=3) or ()) + if complete: + return finish(verdict, selector, entry) + return finish( + _unknown( + verdict.version, + verdict.minimum, + verdict.advisory, + f"Every SNAP-4 version read meets its train's fix, but not every DPU was read: {reason or state}", + ), + None, + entry, + ) + if state == "not_applicable": + verdict = Verdict( + "not-installed", + "not_applicable", + minimum_text, + advisory, + # The standalone harness says why it skipped the scan; a fleet + # scan that found no BlueField-3 says that instead. + "No BlueField-3 DPU is present in the completed scan, so SNAP-4 does not apply" + if not reason or reason.startswith("no BlueField-3") + else f"SNAP-4 does not apply: {reason}", + ) + elif state == "not_installed": + verdict = Verdict( + "not-installed", + "not_applicable", + minimum_text, + advisory, + "Every reached BlueField-3 DPU runs no SNAP-4 container and no kubelet manifest names one", + ) + elif state == "not_running": + verdict = Verdict( + "not-running", + "not_applicable", + minimum_text, + advisory, + "BlueField-3 is in NIC mode, so the DPU Arm cores that run SNAP-4 are off. " + "Recheck SNAP-4 before switching back to DPU mode.", + ) + else: + verdict = _unknown( + "unknown", + minimum_text, + advisory, + reason or "No DPU reported a SNAP-4 container version", + ) + return finish(verdict, None, None) + + def _meets_ladder_minimum(version: str, observed: tuple[int, ...], minimum_version: str) -> bool: """Compare one observed release against its branch minimum in the ladder. @@ -814,6 +1007,26 @@ def _virtio_running_verdict(version: str, *, line: str | None = None) -> Verdict "mlxprivhost -d r --disable_rshim --disable_tracer " "--disable_counter_rd --disable_port_owner" ) +# NIC mode is host-trusted by design, and NVIDIA does not support the zero-trust +# (mlxprivhost restricted) profile there: "Operating in NIC Mode with Zero Trust +# (Restricted) configuration is not supported and may lead to undefined +# behavior" (BlueField BSP 4.10). The supported NIC-mode restriction is the +# INTERNAL_CPU_RSHIM firmware parameter, which takes effect after a power cycle. +NIC_MODE_RSHIM_REMEDIATION = ( + "mlxconfig -d s INTERNAL_CPU_RSHIM=1, then a cold reboot " + "(power cycle) of the host" +) +# INTERNAL_CPU_RSHIM is itself just a card setting, so a tenant with host root +# can set it back. It is real isolation only when the host that gets sudo is the +# only one that can reach the card. Where a tenant has that privilege, the two +# real fixes are on the provider side: keep the tenant's host root off the +# BlueField (sudo -M, mdev-coldplug or an IOMMU/ACS isolation of the PCI +# function), or re-image the card from a known-good BFB before each new tenant +# and after each one leaves. +NIC_MODE_TENANT_SUDO_REMEDIATION = ( + "Keep tenant host root off the BlueField, or re-image the card from a " + "known-good BFB before each new tenant and after each one leaves" +) DPU_ISOLATION_REQUIREMENT = ( "RShim restricted (INTERNAL_CPU_RSHIM=1), no /dev/rshim0, no tmfifo_net0" ) @@ -1365,6 +1578,13 @@ def dpu_host_isolation_verdict(evidence: dict[str, Any] | None = None) -> Verdic # `restricted is False` is deliberately NOT part of that proof: it comes # from mlxconfig reading INTERNAL_CPU_RSHIM, which needs the device the scan # finds, so it cannot be trusted ahead of the scan. It keeps its place below. + nic_mode = str(facts.get("platformMode") or "").strip().lower() == "nic" + remedy = ( + f"Restrict RShim with `{NIC_MODE_RSHIM_REMEDIATION}`; NVIDIA does not " + "support the zero-trust (mlxprivhost) profile in NIC mode." + if nic_mode + else f"Provider policy may require `{MLXPRIVHOST_REMEDIATION}`." + ) if reachable: verdict = Verdict( observed, @@ -1374,8 +1594,7 @@ def dpu_host_isolation_verdict(evidence: dict[str, Any] | None = None) -> Verdic f"The host exposes management interface paths ({', '.join(reachable)}), " f"which fails the zero-trust no-host-interface policy. This does not prove usable " f"RShim access or DPU login. Verify device permissions, SECURE_NIC_MODE, " - f"DROP_MODE and RShim ownership before remediation. Provider policy " - f"may require `{MLXPRIVHOST_REMEDIATION}`.", + f"DROP_MODE and RShim ownership before remediation. {remedy}", ) if not scan_complete: # Name the gap without weakening the grade. The path is proven; what @@ -1408,6 +1627,33 @@ def dpu_host_isolation_verdict(evidence: dict[str, Any] | None = None) -> Verdic "No BlueField DPU is present in the completed device scan", ) + if restricted is False and nic_mode: + # NIC mode is host-trusted by design, but a tenant with host root can + # still load the RShim driver and open the channel: the RShim PCIe + # function is exposed and nothing in firmware blocks it. Through it the + # tenant can push a boot stream that outlives their tenancy, so this + # stays a fail. Only the fix differs, because NVIDIA does not support the + # zero-trust (mlxprivhost) profile in NIC mode. + return _isolation_coverage_caveat( + Verdict( + observed, + "fail", + requirement, + advisory, + "The BlueField is in NIC mode with INTERNAL_CPU_RSHIM=0, so the " + "firmware lets the host open RShim. No /dev/rshim0 or tmfifo_net0 " + "is present, so no RShim driver has opened it yet, but a host root " + "can load one and reach the DPU boot and console interface. No DPU " + "login or escape was attempted. NVIDIA does not support the " + "zero-trust (mlxprivhost) profile in NIC mode, so restrict RShim " + f"with `{NIC_MODE_RSHIM_REMEDIATION}`. That only isolates the " + "card if a tenant's host root cannot re-open it: the setting is " + "card state, so a tenant with the same privilege can set it " + "back. Where tenants get host root, use " + f"`{NIC_MODE_TENANT_SUDO_REMEDIATION}` instead.", + ), + facts.get("unassessedHosts"), + ) if restricted is False: return _isolation_coverage_caveat( Verdict( @@ -1474,7 +1720,12 @@ def dpu_host_isolation_result(evidence: dict[str, Any] | None = None) -> dict[st for host in (facts.get("unassessedHosts") or []) if str(host).strip() ], - "remediation": MLXPRIVHOST_REMEDIATION, + "platformMode": facts.get("platformMode") or "unknown", + "remediation": ( + NIC_MODE_RSHIM_REMEDIATION + if str(facts.get("platformMode") or "").strip().lower() == "nic" + else MLXPRIVHOST_REMEDIATION + ), } @@ -1739,6 +1990,7 @@ def evaluate( nvidia_gpu_present: bool | None = None, amd_model: str = "unknown", amd_driver_evidence: dict[str, Any] | None = None, + snap4: dict[str, Any] | None = None, ) -> dict[str, object]: if gpu_vendor.lower() == "nvidia": driver_result = nvidia_driver_verdict(driver) @@ -1848,6 +2100,7 @@ def evaluate( "dcgmExporter": _verdict_record( dcgm_exporter_result, "dcgmExporter" ), + "snap4Container": snap4_container_result(snap4), "floorsMetadata": minimums_metadata(), } @@ -1942,6 +2195,11 @@ def main() -> int: help="virtio-net check record JSON carrying scanComplete, bluefield3Present, " "and rshimHostAccess", ) + parser.add_argument( + "--snap4-json", + default=None, + help="the virtio-net check's SNAP-4 rollup (state, reason, observed)", + ) args = parser.parse_args() print( json.dumps( @@ -1971,6 +2229,7 @@ def main() -> int: nvidia_gpu_present=False if args.nvidia_gpu_absent else None, dcgm=args.dcgm, dcgm_exporter=args.dcgm_exporter, + snap4=_parse_isolation_json(args.snap4_json), ), sort_keys=True, ) diff --git a/cmax/security.py b/cmax/security.py index 969d42d..31eb579 100644 --- a/cmax/security.py +++ b/cmax/security.py @@ -1549,6 +1549,57 @@ def _virtio_net_bluefield(audit: dict[str, Any]) -> tuple[str, str, str]: ) +def _snap4_container(audit: dict[str, Any]) -> tuple[str, str, str]: + """Grade the BlueField SNAP-4 container against its release-train minimum. + + SNAP-4 runs on the DPU Arm cores, so the audit reads it on the same DPU + path as the virtio-net controller. The audit already graded every reading + against its own train; this only words the result. + """ + verdict = _get(audit, "securityVersions.snap4Container") + if not isinstance(verdict, dict): + return ( + WARNING, + "not collected", + "The BlueField SNAP-4 read was unavailable, so exposure cannot be ruled out.", + ) + status = str(verdict.get("status") or "unknown").lower() + version = _display(verdict.get("version")) + minimum = _display(verdict.get("minimum")) + detail = str(verdict.get("detail") or "").strip() + if status in {"not_applicable", "not-applicable"}: + state = str(verdict.get("state") or "") + observed = { + "not_installed": "no SNAP-4 container on any reached DPU", + "not_running": "no SNAP-4 running; card in NIC mode", + }.get(state, "no BlueField DPU present") + return NOT_APPLICABLE, observed, detail or "SNAP-4 does not apply to this cluster." + if status == "pass": + grace_active, grace_note = _grace_details(verdict) + if grace_active: + return ( + PASS, + f"observed version {version}; minimum version {minimum} during vendor update window", + detail or grace_note, + ) + return ( + PASS, + f"observed version {version}; minimum version {minimum}", + "The SNAP-4 container is at or above the fix for its release train.", + ) + if status == "fail": + return ( + CRITICAL, + f"observed version {version}; minimum version {minimum}", + detail or "The SNAP-4 container is below the fix for its release train.", + ) + return ( + WARNING, + f"observed version {version}; minimum version {minimum}", + detail or "The BlueField SNAP-4 container version requires provider attestation.", + ) + + def _dpu_host_isolation(audit: dict[str, Any]) -> tuple[str, str, str]: """Grade whether the host can reach the DPU control plane over RShim. @@ -1579,6 +1630,20 @@ def _dpu_host_isolation(audit: dict[str, Any]) -> tuple[str, str, str]: f"RShim restricted ({observed})", "The host cannot reach the DPU control plane over RShim.", ) + if status == "fail" and str(verdict.get("platformMode") or "").lower() == "nic": + # NVIDIA does not support the zero-trust mlxprivhost profile in NIC + # mode, so the fix is the INTERNAL_CPU_RSHIM firmware restriction. + return ( + CRITICAL, + f"RShim reachable from the host, NIC mode ({observed})", + "The BlueField is in NIC mode and its firmware lets the host open " + "RShim, so a tenant with host root can load the RShim driver and " + "reach the DPU boot and console interface. Keep tenant host root off " + "the BlueField, or re-image the card from a known-good BFB per " + "tenant; `mlxconfig -d s INTERNAL_CPU_RSHIM=1` and a power " + "cycle is real only when a tenant cannot set it back. Do not apply " + "mlxprivhost zero-trust in NIC mode.", + ) if status == "fail": return ( CRITICAL, @@ -2242,9 +2307,13 @@ def _observability(audit: dict[str, Any]) -> tuple[str, str, str]: "RShim is enabled by default, so the host reaches the DPU Arm cores " "directly. On a multi-tenant cluster a tenant with host root then " "reaches a control plane outside the hypervisor boundary.", - "Apply the zero-trust host profile: `mlxprivhost -d r " - "--disable_rshim --disable_tracer --disable_counter_rd " - "--disable_port_owner`.", + # NVIDIA does not support the zero-trust profile in NIC mode, so the + # static fix names both modes, matching the per-verdict assessment. + "In DPU mode, apply the zero-trust host profile: `mlxprivhost -d " + " r --disable_rshim --disable_tracer --disable_counter_rd " + "--disable_port_owner`. In NIC mode, do not apply mlxprivhost " + "zero-trust. The provider must enforce RShim isolation so tenant root " + "cannot reverse it, or re-image from a trusted BFB between tenants.", _references( ( "NVIDIA mlxprivhost host restriction", @@ -2257,6 +2326,24 @@ def _observability(audit: dict[str, Any]) -> tuple[str, str, str]: ), _dpu_host_isolation, ), + CheckSpec( + "snap4-container", + "BlueField SNAP-4 container", + "A SNAP-4 container below the fix for its release train exposes the " + "DPU storage emulation service to the flaws in NVIDIA's SNAP-4 bulletin.", + "Update the SNAP-4 container to the fixed release for its train, or to " + "a newer train. A card in NIC mode is not running SNAP-4 now, and a " + "switch to DPU mode starts it.", + # The bulletin link and CVEs come from the minimum table (see + # MINIMUM_BOUND_CHECKS). + _references( + ( + "NVIDIA product security", + "https://www.nvidia.com/en-us/security/", + ), + ), + _snap4_container, + ), ) @@ -2313,6 +2400,14 @@ def _observability(audit: dict[str, Any]) -> tuple[str, str, str]: "switch to DPU mode activates it.", "NVIDIA BlueField VIRTIO-Net bulletin", ), + "snap4-container": ( + "snap4Container", + # Per release train, so no single minimum to name. + "Update the SNAP-4 container to the fixed release for its train, or to " + "a newer train. A card in NIC mode is not running SNAP-4 now, and a " + "switch to DPU mode starts it.", + "NVIDIA Networking SNAP4 bulletin", + ), } @@ -2519,6 +2614,7 @@ def evaluate_security(values: dict[str, Any]) -> list[SecurityCheck]: "docker": "securityVersions.docker", "connectx-firmware": "securityVersions.connectxFirmware", "virtio-net-bluefield": "securityVersions.virtioNetBluefield", + "snap4-container": "securityVersions.snap4Container", "fragnesia": "security.fragnesia", "guest-kernel": "security.guestKernel.ubuntuCveGracePeriods", } diff --git a/tests/audit/test_audit_findings.py b/tests/audit/test_audit_findings.py index e4a0b6c..1e59378 100644 --- a/tests/audit/test_audit_findings.py +++ b/tests/audit/test_audit_findings.py @@ -1180,6 +1180,47 @@ def test_reachable_dpu_control_plane_names_the_remediation(self) -> None: self.assertIn("mlxprivhost -d 03:00.0", finding.recommendation) self.assertIn("INTERNAL_CPU_RSHIM=1", finding.recommendation) + def test_nic_mode_rshim_fail_names_the_nic_mode_fix(self) -> None: + keyed = by_key( + self.detect( + { + "dpuHostIsolation": { + "status": "fail", + "bluefieldPresent": True, + "platformMode": "nic", + "detail": "The BlueField is in NIC mode with INTERNAL_CPU_RSHIM=0.", + "remediation": "mlxconfig -d s INTERNAL_CPU_RSHIM=1", + } + } + ) + ) + finding = keyed["securityVersions.dpuHostIsolation.status"] + self.assertEqual(finding.severity, audit_findings.CONFIG) + self.assertIn("NIC mode", finding.title) + self.assertIn("re-image", finding.title) + self.assertNotIn("--disable_rshim", finding.title) + self.assertIn("re-image it from a known-good BFB", finding.recommendation) + self.assertIn("off the BlueField", finding.recommendation) + self.assertIn("does not support the zero-trust", finding.recommendation) + self.assertIn("real only under the first path", finding.recommendation) + + def test_open_rshim_path_in_nic_mode_recommends_the_nic_mode_fix(self) -> None: + keyed = by_key( + self.detect( + { + "dpuHostIsolation": { + "status": "fail", + "bluefieldPresent": True, + "platformMode": "nic", + "remediation": "mlxconfig -d s INTERNAL_CPU_RSHIM=1", + } + } + ) + ) + finding = keyed["securityVersions.dpuHostIsolation.status"] + self.assertIn("re-image it from a known-good BFB", finding.recommendation) + self.assertIn("real only under the first path", finding.recommendation) + def test_perf_controls_name_values_and_persistent_actions(self) -> None: keyed = by_key( audit_findings.detect_findings( diff --git a/tests/audit/test_minimum_versions_shell.py b/tests/audit/test_minimum_versions_shell.py index 39bc7ba..928207d 100644 --- a/tests/audit/test_minimum_versions_shell.py +++ b/tests/audit/test_minimum_versions_shell.py @@ -1658,6 +1658,7 @@ def test_every_virtio_net_flag_is_emitted_with_its_value(self) -> None: "virtioNetWorstObservedMode": "dpu", "virtioNetWorstObservedHost": "worker-3", "virtioNetObservedJson": '[{"host": "worker-3"}]', + "snap4Json": '{"state": "version", "observed": [{"version": "4.9.0"}]}', } ) @@ -1674,6 +1675,9 @@ def test_every_virtio_net_flag_is_emitted_with_its_value(self) -> None: self.assertEqual(pairs["--virtio-net-worst-host"], "worker-3") self.assertEqual(pairs["--virtio-net-observed-json"], '[{"host": "worker-3"}]') self.assertEqual(pairs["--dpu-isolation-json"], '{"scanComplete": true}') + self.assertEqual( + pairs["--snap4-json"], '{"state": "version", "observed": [{"version": "4.9.0"}]}' + ) class DegradedSecurityAuditFallbackTests(TempDirMixin): def evaluator_components(self) -> set[str]: diff --git a/tests/audit/test_security_cli.py b/tests/audit/test_security_cli.py index d3e7f0f..2365478 100644 --- a/tests/audit/test_security_cli.py +++ b/tests/audit/test_security_cli.py @@ -40,6 +40,7 @@ "docker": "https://example.invalid/minimums/docker", "connectxFirmware": "https://example.invalid/minimums/connectx", "virtioNetBluefield": "https://example.invalid/minimums/virtio", + "snap4Container": "https://example.invalid/minimums/snap4", } @@ -91,6 +92,13 @@ def minimum_table(generated: datetime, **components: dict) -> dict: "cves": ["CVE-2026-65094"], "source": {"aId": 5815, "feed": "nvidia-csaf"}, }, + "snap4Container": { + "kind": "releaseLines", + "lines": {"4.5": {"fixed": "4.5.5"}, "4.9": {"fixed": "4.9.1"}}, + "advisory": FIXTURE_ADVISORIES["snap4Container"], + "cves": ["CVE-2025-33215", "CVE-2025-33216"], + "source": {"aId": 5744, "feed": "nvidia-csaf"}, + }, }, } @@ -866,7 +874,7 @@ def test_default_report_shows_every_check_and_observed_versions(self) -> None: Path("/tmp/report"), ) self.assertIn("# ClusterMAX security audit report", report) - self.assertIn("3 failed, 12 warnings, 4 passed, 3 skipped", report) + self.assertIn("3 failed, 13 warnings, 4 passed, 3 skipped", report) self.assertIn("Action required: 3 critical findings.", report) self.assertNotIn("critical findings are known exposure", report) self.assertIn("NVIDIA driver minimum version", report) @@ -1314,6 +1322,12 @@ def test_a_present_device_still_passes_on_its_own_merits(self) -> None: ) self.assertEqual(status, security.PASS) + def test_remediation_does_not_prescribe_zero_trust_in_nic_mode(self) -> None: + spec = next(s for s in security.CHECK_SPECS if s.id == "dpu-host-isolation") + self.assertIn("In DPU mode, apply the zero-trust host profile", spec.remediation) + self.assertIn("In NIC mode, do not apply mlxprivhost zero-trust", spec.remediation) + self.assertIn("re-image from a trusted BFB", spec.remediation) + def test_absent_hardware_is_given_no_remediation_to_apply(self) -> None: checks = [ security.SecurityCheck( @@ -1407,12 +1421,12 @@ def test_staleness_changes_no_count_and_no_exit_code(self) -> None: security.counts(stale_checks), { security.PASS: 3, - security.WARNING: 12, + security.WARNING: 13, security.CRITICAL: 4, security.NOT_APPLICABLE: 3, }, ) - self.assertEqual(sum(security.counts(stale_checks).values()), 22) + self.assertEqual(sum(security.counts(stale_checks).values()), 23) self.assertEqual(clean_exit, 0) report = self._report(now=self.generated + timedelta(days=40), verbosity=3) self.assertNotIn("minimum-version-freshness", report) @@ -1467,8 +1481,8 @@ def test_saved_report_records_the_minimum_state(self) -> None: def test_minimum_freshness_is_not_a_graded_criterion(self) -> None: ids = [spec.id for spec in security.CHECK_SPECS] - self.assertEqual(len(ids), 22) - self.assertEqual(len(set(ids)), 22) + self.assertEqual(len(ids), 23) + self.assertEqual(len(set(ids)), 23) self.assertEqual( ids, [ @@ -1494,6 +1508,7 @@ def test_minimum_freshness_is_not_a_graded_criterion(self) -> None: "nvlink-boundary", "virtio-net-bluefield", "dpu-host-isolation", + "snap4-container", ], ) diff --git a/tests/audit/test_security_version_audit.py b/tests/audit/test_security_version_audit.py index 60ae888..60c16db 100644 --- a/tests/audit/test_security_version_audit.py +++ b/tests/audit/test_security_version_audit.py @@ -1162,6 +1162,7 @@ class VirtioNetRollupReachesTheVerdictTests(unittest.TestCase): "virtioNetWorstObservedHost": "--virtio-net-worst-host", "virtioNetObservedJson": "--virtio-net-observed-json", "dpuIsolationJson": "--dpu-isolation-json", + "snap4Json": "--snap4-json", } @classmethod @@ -1510,6 +1511,58 @@ def test_each_host_side_path_to_the_dpu_control_plane_fails(self) -> None: self.assertIn("mlxprivhost", verdict.detail) self.assertIn("--disable_rshim", verdict.detail) + def test_nic_mode_with_rshim_enabled_fails_with_the_nic_mode_fix(self) -> None: + # NVIDIA does not support the zero-trust (mlxprivhost) profile in NIC + # mode, so the finding must not tell the provider to apply it there. + evidence = { + **self.evidence(rshimRestricted=False, internalCpuRshim="0"), + "platformMode": "nic", + } + verdict = security.dpu_host_isolation_verdict(evidence) + # A host root can still load the RShim driver and open the channel. + self.assertEqual(verdict.status, "fail") + self.assertIn("INTERNAL_CPU_RSHIM=1", verdict.detail) + self.assertIn("power cycle", verdict.detail) + self.assertIn("does not support the zero-trust", verdict.detail) + self.assertNotIn("--disable_rshim", verdict.detail) + self.assertIn("can set it back", verdict.detail) + self.assertIn("re-image the card", verdict.detail) + self.assertIn("off the BlueField", verdict.detail) + result = security.dpu_host_isolation_result(evidence) + self.assertEqual(result["remediation"], security.NIC_MODE_RSHIM_REMEDIATION) + self.assertEqual(result["platformMode"], "nic") + + def test_an_open_rshim_path_in_nic_mode_still_fails(self) -> None: + evidence = { + **self.evidence(rshimRestricted=False, rshimDeviceNode=True), + "platformMode": "nic", + } + verdict = security.dpu_host_isolation_verdict(evidence) + self.assertEqual(verdict.status, "fail") + self.assertIn("INTERNAL_CPU_RSHIM=1", verdict.detail) + self.assertNotIn("--disable_rshim", verdict.detail) + + def test_dpu_or_unknown_mode_with_rshim_enabled_still_fails(self) -> None: + for mode in ("dpu", "unknown", None): + with self.subTest(mode=mode): + evidence = { + **self.evidence(rshimRestricted=False, internalCpuRshim="0"), + "platformMode": mode, + } + verdict = security.dpu_host_isolation_verdict(evidence) + self.assertEqual(verdict.status, "fail") + self.assertIn("mlxprivhost", verdict.detail) + + def test_a_nic_mode_fail_names_unassessed_hosts(self) -> None: + evidence = { + **self.evidence(rshimRestricted=False, internalCpuRshim="0"), + "platformMode": "nic", + "unassessedHosts": ["worker-9"], + } + verdict = security.dpu_host_isolation_verdict(evidence) + self.assertEqual(verdict.status, "fail") + self.assertIn("worker-9", verdict.detail) + def test_a_missing_pci_scan_cannot_erase_a_proven_rshim_path(self) -> None: for label, paths in ( ("device node", {"rshimDeviceNode": True, "tmfifoNet0": False}), diff --git a/tests/audit/test_snap4_container.py b/tests/audit/test_snap4_container.py new file mode 100644 index 0000000..65ca471 --- /dev/null +++ b/tests/audit/test_snap4_container.py @@ -0,0 +1,389 @@ +"""The BlueField SNAP-4 container read and its per-train grade. + +SNAP-4 runs on the DPU Arm cores, so the virtio-net check reads it on the same +DPU path. These tests pin the per-host states, the cluster rollup, and the +security verdict, including the per-train rule from NVIDIA bulletin 5744. + +Sources: cmax/scripts/1-audit/checks/fabric/virtio-net-check.py (the read and +rollup) and cmax/scripts/1-audit/security_version_audit.py (the grade). +""" + +from __future__ import annotations + +import json +import os +import shutil +import unittest +from pathlib import Path + +from tests.audit.test_security_version_audit import security +from tests.audit.test_virtio_net_check import ( + BLUEFIELD3_LSPCI, + DPU_MODE_MLXCONFIG, + NIC_MODE_MLXCONFIG, + NO_BLUEFIELD_LSPCI, + VIRTNET_VERSION_JSON, + CheckHarness, + load_check_module, +) + +minimums = security.minimum_versions + +SNAP_IMAGE_ID = "sha256:5d3f0b2c9a" +RUNNING_PS = json.dumps( + { + "containers": [ + { + "id": "c1", + "metadata": {"name": "doca-snap"}, + "image": {"image": SNAP_IMAGE_ID}, + "imageRef": SNAP_IMAGE_ID, + "state": "CONTAINER_RUNNING", + }, + { + "id": "c2", + "metadata": {"name": "doca-telemetry"}, + "image": {"image": "sha256:other"}, + "imageRef": "sha256:other", + }, + ] + }, + indent=2, +) +IMAGES = json.dumps( + { + "images": [ + {"id": SNAP_IMAGE_ID, "repoTags": ["nvcr.io/nvidia/doca/doca_snap:4.9.0-doca3.1.0"]}, + {"id": "sha256:other", "repoTags": ["nvcr.io/nvidia/doca/doca_telemetry:1.21.4-doca3.1.0"]}, + ] + }, + indent=2, +) + + +def listing(*, manifests: str = "", ps: str | None = RUNNING_PS, images: str = IMAGES) -> str: + """What SNAP4_DPU_SCRIPT prints on a DPU.""" + lines = ["@@SNAP4-MANIFESTS@@", manifests] + if ps is None: + lines += ["@@SNAP4-NO-CRICTL@@"] + else: + lines += ["@@SNAP4-PS@@", ps, "@@SNAP4-IMAGES@@", images, "@@SNAP4-END@@"] + return "\n".join(lines) + "\n" + + +class Snap4ListingParseTests(unittest.TestCase): + def setUp(self) -> None: + self.check = load_check_module() + + def test_running_container_version_comes_from_the_image_tag(self) -> None: + parsed = self.check.parse_snap4_listing(listing()) + self.assertTrue(parsed["reached"]) + self.assertTrue(parsed["listed"]) + self.assertEqual(parsed["running"], ["4.9.0"]) + + def test_container_named_by_its_image_needs_no_images_listing(self) -> None: + ps = json.dumps( + {"containers": [{"metadata": {"name": "x"}, "image": {"image": "nvcr.io/nvidia/doca/doca_snap:4.5.5-doca2.9.1"}}]} + ) + parsed = self.check.parse_snap4_listing(listing(ps=ps, images="{}")) + self.assertEqual(parsed["running"], ["4.5.5"]) + + def test_manifest_version_is_read_when_nothing_runs(self) -> None: + parsed = self.check.parse_snap4_listing( + listing( + manifests=" image: nvcr.io/nvidia/doca/doca_snap:4.9.1-doca3.2.0", + ps=json.dumps({"containers": []}), + ) + ) + self.assertEqual(parsed["running"], []) + self.assertEqual(parsed["configured"], ["4.9.1"]) + + def test_a_snap_container_without_a_tag_is_flagged(self) -> None: + ps = json.dumps({"containers": [{"metadata": {"name": "snap"}, "image": {"image": "sha256:none"}}]}) + parsed = self.check.parse_snap4_listing(listing(ps=ps, images="{}")) + self.assertEqual(parsed["running"], []) + self.assertTrue(parsed["runningWithoutVersion"]) + + def test_no_crictl_and_no_output_are_not_a_listing(self) -> None: + self.assertFalse(self.check.parse_snap4_listing(listing(ps=None))["listed"]) + empty = self.check.parse_snap4_listing("") + self.assertFalse(empty["reached"]) + self.assertFalse(empty["listed"]) + + def test_failed_ps_is_not_a_listing(self) -> None: + text = "@@SNAP4-MANIFESTS@@\n@@SNAP4-PS@@\n@@SNAP4-PS-FAILED@@\n" + parsed = self.check.parse_snap4_listing(text) + self.assertTrue(parsed["reached"]) + self.assertFalse(parsed["listed"]) + + +class Snap4HostReadTests(CheckHarness): + def bluefield(self, mode: str = DPU_MODE_MLXCONFIG) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + self.stub_output("mlxconfig", mode) + (self.root / "dev" / "rshim0").touch() + + def ssh_prints(self, text: str, *, returncode: int = 0) -> None: + data = self.tmp / "ssh-snap4.stdout" + data.write_text(text) + # virtnet over SSH fails; the SNAP-4 script (the argument carrying + # crictl) prints the canned DPU listing. + self.stub( + "ssh", + f""" +for arg in "$@"; do + case "$arg" in + *crictl*) /bin/cat "{data}"; exit {returncode} ;; + esac +done +exit 1 +""", + ) + + def snap_calls(self) -> str: + """The SSH call that carried the SNAP-4 script, or "" if none did.""" + path = self.calls / "ssh.calls" + text = path.read_text() if path.exists() else "" + return text if "crictl" in text else "" + + def test_running_container_is_read_over_rshim(self) -> None: + self.bluefield() + self.ssh_prints(listing()) + + record = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="") + + snap = record["snap4"] + self.assertEqual(snap["state"], "version") + self.assertEqual(snap["versions"], ["4.9.0"]) + self.assertEqual(snap["source"], "rshim-ssh/crictl") + self.assertTrue(record["rshimHostAccess"]["dpuReachedFromHost"]) + argv = self.snap_calls() + self.assertIn("BatchMode=yes", argv) + self.assertIn("sudo -n crictl", argv) + # The read lists only. It never pulls, runs, stops, or execs. + for verb in ("crictl pull", "crictl run", "crictl stop", "crictl exec", "crictl rm"): + self.assertNotIn(verb, argv) + + def test_collector_summary_carries_the_rollup(self) -> None: + self.bluefield() + self.ssh_prints(listing()) + + summary = self.run_check("--summary", CLUSTERMAX_DPU_SSH_DISABLE="") + + snap = json.loads(summary["snap4Json"]) + self.assertEqual(snap["state"], "version") + self.assertEqual(snap["observed"][0]["version"], "4.9.0") + + def test_reached_dpu_without_snap_is_not_installed(self) -> None: + self.bluefield() + self.ssh_prints(listing(ps=json.dumps({"containers": []}))) + + snap = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="")["snap4"] + + self.assertEqual(snap["state"], "not_installed") + self.assertEqual(snap["versions"], []) + + def test_unreachable_dpu_is_unknown_and_asks_for_attestation(self) -> None: + self.bluefield() + self.stub("ssh", 'echo "Permission denied (publickey)." >&2\nexit 255') + + snap = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="")["snap4"] + + self.assertEqual(snap["state"], "unknown") + self.assertIn("attest", snap["reason"]) + + def test_disabled_ssh_is_unknown_without_a_call(self) -> None: + self.bluefield() + self.ssh_prints(listing()) + + snap = self.host_record()["snap4"] + + self.assertEqual(snap["state"], "unknown") + self.assertEqual(self.snap_calls(), "") + + def test_nic_mode_is_not_running_and_never_dials_the_dpu(self) -> None: + self.bluefield(NIC_MODE_MLXCONFIG) + self.ssh_prints(listing()) + + snap = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="")["snap4"] + + self.assertEqual(snap["state"], "not_running") + self.assertEqual(self.snap_calls(), "") + + def test_no_bluefield_is_not_applicable(self) -> None: + self.stub_output("lspci", NO_BLUEFIELD_LSPCI) + self.ssh_prints(listing()) + + snap = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="")["snap4"] + + self.assertEqual(snap["state"], "not_applicable") + self.assertEqual(self.snap_calls(), "") + + def test_on_the_dpu_the_read_runs_locally(self) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + self.stub_output("mlxconfig", DPU_MODE_MLXCONFIG) + payload = self.tmp / "virtnet-version.json" + payload.write_text(VIRTNET_VERSION_JSON) + self.stub("virtnet", f'case "$1" in version) /bin/cat "{payload}" ;; *) exit 1 ;; esac') + for tool in ("sh", "grep"): + os.symlink(shutil.which(tool), self.bin / tool) + (self.tmp / "ps.json").write_text(RUNNING_PS) + (self.tmp / "images.json").write_text(IMAGES) + self.stub( + "crictl", + f'case "$1" in ps) /bin/cat "{self.tmp}/ps.json" ;; images) /bin/cat "{self.tmp}/images.json" ;; esac', + ) + self.ssh_prints(listing()) + + snap = self.host_record(CLUSTERMAX_DPU_SSH_DISABLE="")["snap4"] + + self.assertEqual(snap["versions"], ["4.9.0"]) + self.assertEqual(snap["source"], "local/crictl") + self.assertEqual(self.snap_calls(), "") + + +class Snap4SummaryTests(unittest.TestCase): + def setUp(self) -> None: + self.check = load_check_module() + + def record(self, host: str, state: str, versions=(), source="rshim-ssh/crictl", reason="r") -> dict: + return { + "host": host, + "state": "version", + "snap4": {"state": state, "versions": list(versions), "source": source, "reason": reason}, + } + + def test_every_reading_survives_the_rollup(self) -> None: + summary = self.check.summarize_snap4( + [ + self.record("a", "version", ["4.9.1"]), + self.record("b", "version", ["4.7.0"]), + self.record("c", "unknown", reason="DPU not reached"), + ] + ) + self.assertEqual(summary["state"], "unknown") + self.assertEqual([entry["version"] for entry in summary["observed"]], ["4.7.0", "4.9.1"]) + self.assertEqual(summary["hostsChecked"], 3) + + def test_nic_peer_outranks_a_read_host(self) -> None: + summary = self.check.summarize_snap4( + [self.record("a", "version", ["4.9.1"]), self.record("b", "not_running")] + ) + self.assertEqual(summary["state"], "not_running") + + def test_a_coverage_gap_withdraws_a_clean_rollup(self) -> None: + summary = {"snap4": {"state": "not_installed", "reason": "", "observed": []}} + gapped = self.check.apply_coverage_gap(dict(summary), ["node-7: srun failed"]) + self.assertEqual(gapped["snap4"]["state"], "incomplete") + + +SNAP4_BLOCK = { + "kind": "releaseLines", + "lines": {"4.5": {"fixed": "4.5.5"}, "4.9": {"fixed": "4.9.1"}}, + "advisory": "https://nvidia.custhelp.com/app/answers/detail/a_id/5744", + "cves": ["CVE-2025-33215", "CVE-2025-33216"], + "source": {"aId": 5744, "feed": "nvidia-csaf"}, +} + + +class Snap4VerdictTests(unittest.TestCase): + def setUp(self) -> None: + import tempfile + + table = json.loads(minimums.minimums_path().read_text()) + table["components"]["snap4Container"] = SNAP4_BLOCK + holder = tempfile.TemporaryDirectory() + self.addCleanup(holder.cleanup) + path = Path(holder.name) / "minimums.json" + path.write_text(json.dumps(table)) + previous = os.environ.get(minimums.MINIMUMS_ENV) + os.environ[minimums.MINIMUMS_ENV] = str(path) + minimums._CACHE = None + + def restore() -> None: + if previous is None: + os.environ.pop(minimums.MINIMUMS_ENV, None) + else: + os.environ[minimums.MINIMUMS_ENV] = previous + minimums._CACHE = None + + self.addCleanup(restore) + + def grade(self, version: str) -> tuple[str, str]: + verdict, _ = security.snap4_verdict(version) + return verdict.status, verdict.minimum + + def test_each_train_is_held_to_its_own_fix(self) -> None: + self.assertEqual(self.grade("4.9.0"), ("fail", "4.9.1")) + self.assertEqual(self.grade("4.9.1"), ("pass", "4.9.1")) + self.assertEqual(self.grade("4.5.4"), ("fail", "4.5.5")) + self.assertEqual(self.grade("4.5.5"), ("pass", "4.5.5")) + + def test_a_train_without_a_fix_moves_to_the_next_fix_above(self) -> None: + self.assertEqual(self.grade("4.7.2"), ("fail", "4.9.1")) + self.assertEqual(self.grade("4.3.0"), ("fail", "4.5.5")) + + def test_a_train_newer_than_every_fix_passes(self) -> None: + self.assertEqual(self.grade("4.12.0"), ("pass", "4.9.1")) + + def result(self, state: str, *observed: tuple[str, str], reason: str = "") -> dict: + return security.snap4_container_result( + { + "state": state, + "reason": reason, + "observed": [{"version": v, "host": h, "source": "rshim-ssh/crictl"} for v, h in observed], + } + ) + + def test_the_worst_reading_decides(self) -> None: + result = self.result("version", ("4.12.0", "a"), ("4.9.0", "b"), ("4.5.5", "c")) + self.assertEqual(result["status"], "fail") + self.assertEqual(result["version"], "4.9.0") + self.assertEqual(result["minimum"], "4.9.1") + self.assertEqual(result["host"], "b") + + def test_a_fail_stands_beside_an_unread_peer(self) -> None: + result = self.result("unknown", ("4.7.0", "a"), reason="DPU not reached") + self.assertEqual(result["status"], "fail") + self.assertIn("Not every DPU was read", result["detail"]) + + def test_passing_readings_do_not_clear_unread_peers(self) -> None: + result = self.result("unknown", ("4.9.1", "a"), reason="DPU not reached") + self.assertEqual(result["status"], "unknown") + self.assertIn("not every DPU was read", result["detail"]) + + def test_passing_readings_with_full_coverage_pass(self) -> None: + result = self.result("not_running", ("4.9.1", "a")) + self.assertEqual(result["status"], "pass") + + def test_nothing_to_read_is_not_applicable(self) -> None: + for state in ("not_applicable", "not_installed", "not_running"): + with self.subTest(state=state): + self.assertEqual(self.result(state)["status"], "not_applicable") + + def test_unread_dpu_mode_is_unknown(self) -> None: + result = self.result("unknown", reason="the DPU did not answer") + self.assertEqual(result["status"], "unknown") + self.assertIn("did not answer", result["detail"]) + + def test_no_rollup_is_unknown(self) -> None: + self.assertEqual(security.snap4_container_result(None)["status"], "unknown") + + +class Snap4WithoutTableEntryTests(unittest.TestCase): + """The shipped table predates snap4Container until the refresh publishes it.""" + + def test_reading_without_a_minimum_is_unknown_and_absence_still_applies(self) -> None: + table = json.loads(minimums.minimums_path().read_text()) + if "snap4Container" in table.get("components", {}): + self.skipTest("the shipped table already lists snap4Container") + read = security.snap4_container_result( + {"state": "version", "observed": [{"version": "4.9.1", "host": "a", "source": "x"}]} + ) + self.assertEqual(read["status"], "unknown") + absent = security.snap4_container_result({"state": "not_applicable", "observed": []}) + self.assertEqual(absent["status"], "not_applicable") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/audit/test_virtio_net_check.py b/tests/audit/test_virtio_net_check.py index 0c9ca11..5fd94cc 100644 --- a/tests/audit/test_virtio_net_check.py +++ b/tests/audit/test_virtio_net_check.py @@ -357,6 +357,65 @@ def test_missing_lspci_is_incomplete_not_not_applicable(self) -> None: self.assertFalse(record["scanComplete"]) self.assertIsNone(record["bluefield3Present"]) + def sysfs_device(self, slot: str, vendor: str, device: str, pci_class: str) -> Path: + path = self.root / "sys" / "bus" / "pci" / "devices" / slot + path.mkdir(parents=True) + (path / "vendor").write_text(f"0x{vendor}\n") + (path / "device").write_text(f"0x{device}\n") + (path / "class").write_text(f"0x{pci_class}\n") + return path + + def test_missing_lspci_falls_back_to_sysfs_and_finds_a_bluefield(self) -> None: + # Slurm-on-Kubernetes slurmd images ship without pciutils but still + # mount the host's sysfs, which names the same devices. + self.stub_missing("lspci") + self.stub_missing("mlxconfig") + self.sysfs_device("0000:9a:00.0", "15b3", "a2dc", "020000") + self.sysfs_device("0000:17:00.0", "15b3", "1021", "020700") + + record = self.host_record() + + self.assertTrue(record["scanComplete"]) + self.assertEqual(record["pciSource"], "sysfs") + self.assertTrue(record["bluefield3Present"]) + self.assertEqual(record["bluefieldDevices"][0]["slot"], "0000:9a:00.0") + self.assertEqual(record["state"], "incomplete") + + def test_missing_lspci_with_a_bluefield_free_sysfs_is_not_applicable(self) -> None: + self.stub_missing("lspci") + self.sysfs_device("0000:17:00.0", "15b3", "1021", "020700") + self.sysfs_device("0000:00:04.0", "1af4", "1001", "010000") + + record = self.host_record() + + self.assertTrue(record["scanComplete"]) + self.assertEqual(record["pciSource"], "sysfs") + self.assertFalse(record["bluefield3Present"]) + self.assertEqual(record["virtioPciDevices"], 1) + self.assertEqual(record["state"], "not_applicable") + + def test_an_unreadable_sysfs_entry_keeps_the_fallback_incomplete(self) -> None: + self.stub_missing("lspci") + self.sysfs_device("0000:17:00.0", "15b3", "1021", "020700") + broken = self.root / "sys" / "bus" / "pci" / "devices" / "0000:18:00.0" + broken.mkdir(parents=True) + + record = self.host_record() + + self.assertFalse(record["scanComplete"]) + self.assertIsNone(record["bluefield3Present"]) + self.assertIn("sysfs could not read 1", record["scanError"]) + self.assertEqual(record["state"], "incomplete") + + def test_lspci_stays_the_source_when_it_runs(self) -> None: + self.stub_output("lspci", NO_BLUEFIELD_LSPCI) + self.sysfs_device("0000:9a:00.0", "15b3", "a2dc", "020000") + + record = self.host_record() + + self.assertEqual(record["pciSource"], "lspci") + self.assertFalse(record["bluefield3Present"]) + def test_mlxconfig_without_root_is_incomplete_not_not_applicable(self) -> None: self.stub_output("lspci", BLUEFIELD3_LSPCI) @@ -461,6 +520,19 @@ def test_missing_mlxconfig_is_incomplete(self) -> None: self.assertEqual(record["state"], "incomplete") self.assertIn("mlxconfig", record["reason"]) + def test_missing_mlxconfig_is_not_blamed_on_sudo(self) -> None: + # Seen on CoreWeave SUNK: sudo -n works, mlxconfig is simply absent. + self.stub_output("lspci", BLUEFIELD3_LSPCI) + self.stub_missing("mlxconfig") + self.stub_sudo() + + record = self.host_record() + + self.assertEqual(record["state"], "incomplete") + self.assertIn("mlxconfig is not installed", record["reason"]) + self.assertIn("install NVIDIA MFT", record["reason"]) + self.assertNotIn("sudo", record["reason"]) + class EscalationPathTests(unittest.TestCase): def setUp(self) -> None: @@ -572,6 +644,83 @@ def test_failed_mode_read_never_loads_host_kernel_modules(self) -> None: self.assertFalse(self.calls_for("mst")) self.assertFalse(any(call.startswith("-n mst") for call in self.calls_for("sudo"))) +class NamedModeQueryTests(CheckHarness): + """The mode read names its three parameters instead of dumping the card. + + On VP's GB300 BlueField-3 hosts the full dump took 22 s idle, past the 15 s + tool timeout, and 14 of 16 hosts graded the mode unreadable. + """ + + def test_the_mode_read_names_only_the_graded_parameters(self) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + self.stub_output("mlxconfig", NIC_MODE_MLXCONFIG) + + record = self.host_record() + + self.assertEqual(record["mode"], "nic") + calls = self.calls_for("mlxconfig") + self.assertEqual(len(calls), 1) + self.assertTrue( + calls[0].endswith( + "q INTERNAL_CPU_MODEL INTERNAL_CPU_OFFLOAD_ENGINE INTERNAL_CPU_RSHIM" + ), + calls[0], + ) + + def test_firmware_that_rejects_a_name_falls_back_to_the_full_dump(self) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + data = self.tmp / "full.stdout" + data.write_text(NIC_MODE_MLXCONFIG) + self.stub( + "mlxconfig", + 'if [ "${@: -1}" = "q" ]; then /bin/cat "%s"; exit 0; fi\n' + "echo 'Configurations: Default Current Next Boot'\n" + "echo '-E- Unknown Parameter: INTERNAL_CPU_RSHIM'\n" + "exit 3" % data, + ) + self.stub_sudo() + + record = self.host_record() + + self.assertEqual(record["mode"], "nic") + self.assertEqual(record["modeSource"], "mlxconfig") + calls = self.calls_for("mlxconfig") + self.assertEqual(len(calls), 2) + self.assertTrue(calls[1].endswith(" q"), calls[1]) + # The device opened unprivileged, so nothing escalates. + self.assertEqual(self.calls_for("sudo"), []) + + def test_a_parameter_the_firmware_lacks_also_falls_back(self) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + data = self.tmp / "full.stdout" + data.write_text(NIC_MODE_MLXCONFIG) + self.stub( + "mlxconfig", + 'if [ "${@: -1}" = "q" ]; then /bin/cat "%s"; exit 0; fi\n' + "echo 'Configurations: Next Boot'\n" + "echo \"-E- The Device doesn't support INTERNAL_CPU_RSHIM parameter\"\n" + "exit 3" % data, + ) + + record = self.host_record() + + self.assertEqual(record["mode"], "nic") + self.assertEqual(len(self.calls_for("mlxconfig")), 2) + + def test_the_full_dump_is_never_read_after_a_failed_open(self) -> None: + self.stub_output("lspci", BLUEFIELD3_LSPCI) + self.stub( + "mlxconfig", 'echo "-E- Failed to open the device" >&2\nexit 1' + ) + self.stub_sudo_denied() + + record = self.host_record() + + self.assertEqual(record["state"], "incomplete") + calls = self.calls_for("mlxconfig") + self.assertEqual(len(calls), 1) + self.assertFalse(calls[0].endswith(" q")) + class RshimEvidenceTests(CheckHarness): def test_rshim_reachability_is_recorded_as_evidence(self) -> None: @@ -1531,3 +1680,35 @@ def _write_severity_matrix() -> None: _write_severity_matrix() else: unittest.main() + + +class IsolationPlatformModeTests(unittest.TestCase): + """The isolation evidence says NIC only when every unrestricted host is NIC.""" + + @staticmethod + def record(host: str, mode: str, restricted: bool) -> dict: + return { + "host": host, + "mode": mode, + "scanComplete": True, + "bluefield3Present": True, + "rshimHostAccess": { + "rshimRestricted": restricted, + "rshimDeviceNode": False, + "tmfifoNet0": False, + }, + } + + def test_modes(self) -> None: + mod = load_check_module() + cases = { + "all nic": ([("a", "nic", False), ("b", "nic", False)], "nic"), + "one dpu": ([("a", "nic", False), ("b", "dpu", False)], "dpu"), + "one unread": ([("a", "nic", False), ("b", "unknown", False)], "unknown"), + "restricted dpu ignored": ([("a", "nic", False), ("b", "dpu", True)], "nic"), + } + for label, (rows, expected) in cases.items(): + with self.subTest(case=label): + records = [self.record(*row) for row in rows] + self.assertEqual(mod.isolation_evidence(records)["platformMode"], expected) +