From 27d6b0dbd2b5a2113fb8c14923d70fb354d5a0ee Mon Sep 17 00:00:00 2001 From: SecureBot Date: Fri, 11 Sep 2026 16:33:32 -0400 Subject: [PATCH 1/2] tp2: render only the selected HCA functions into B12X_ROCE_HCA --- runtime/profiles/glm53-flash-spark-tp2/README.md | 2 +- runtime/profiles/glm53-flash-spark-tp2/launch.py | 9 +++++++-- .../test_glm53_spark_tp2_profile.py | 9 ++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/runtime/profiles/glm53-flash-spark-tp2/README.md b/runtime/profiles/glm53-flash-spark-tp2/README.md index 1a6caf61..10997936 100644 --- a/runtime/profiles/glm53-flash-spark-tp2/README.md +++ b/runtime/profiles/glm53-flash-spark-tp2/README.md @@ -76,7 +76,7 @@ records the common-source requirements and reference-trial limits. | Scheduler | Eight sequences, 8,192 batched tokens, prefill interval 8 | | Prefill | Token-sharded mHC and recurrent-checkpoint coalescing; sequential KDA projection | | Graphs | `FULL_AND_PIECEWISE`, mode 0; `[1,2,4,8,12,16,20,24,28,32]` | -| Transport | HCA indices 0/2; two RoCEnante paths; 16 MiB all-reduce and all-gather input-shard limits | +| Transport | HCA indices 0/2 (`rocep1s0f0`, `roceP2p1s0f0`); only these two functions are rendered into `B12X_ROCE_HCA`, so the other cage may be uncabled; two RoCEnante paths; 16 MiB all-reduce and all-gather input-shard limits | | NCCL | Verified 2.30.7; eight channels; `=rocep1s0f0,roceP2p1s0f0` | | Multimodal / SparkCache | Four images, zero videos; SparkCache disabled | | Lifecycle | Active 2 GiB host-memory guard; manual create/start; Docker restart `no` | diff --git a/runtime/profiles/glm53-flash-spark-tp2/launch.py b/runtime/profiles/glm53-flash-spark-tp2/launch.py index 46569faf..50127df0 100644 --- a/runtime/profiles/glm53-flash-spark-tp2/launch.py +++ b/runtime/profiles/glm53-flash-spark-tp2/launch.py @@ -35,10 +35,15 @@ def transport_environment(rank: int) -> dict[str, str]: raise ValueError("The profile's transport manifest has changed") indices = transport["local_hca_indices"] inventory = transport["hca_inventory"] + # Expose only the selected functions. The RoCE proxy opens every device in + # B12X_ROCE_HCA and requires each to be ACTIVE, so listing the full inventory + # aborts RoCEnante on a host with one physical DAC (the f1 functions have no + # carrier). Peer-map indices are positions in the rendered list. + selected = [inventory[index] for index in indices] return { - "B12X_ROCE_HCA": ",".join(inventory), + "B12X_ROCE_HCA": ",".join(selected), "B12X_ROCE_PAIR_PATHS": str(transport["path_count"]), - "B12X_ROCE_PEER_HCA_MAP": f"{1 - rank}=" + "/".join(map(str, indices)), + "B12X_ROCE_PEER_HCA_MAP": f"{1 - rank}=" + "/".join(str(i) for i in range(len(selected))), "NCCL_IB_HCA": "=" + ",".join(inventory[index] for index in indices), "NCCL_IB_MERGE_NICS": "0", "NCCL_CROSS_NIC": "1", diff --git a/runtime/profiles/glm53-flash-spark-tp2/test_glm53_spark_tp2_profile.py b/runtime/profiles/glm53-flash-spark-tp2/test_glm53_spark_tp2_profile.py index ea974d1a..8843c3ee 100644 --- a/runtime/profiles/glm53-flash-spark-tp2/test_glm53_spark_tp2_profile.py +++ b/runtime/profiles/glm53-flash-spark-tp2/test_glm53_spark_tp2_profile.py @@ -168,8 +168,11 @@ def value(flag): def test_rank_plan_maps_both_pci_functions_of_one_cage(inputs, rank): value = plan(inputs, rank) env = value["environment"] - assert env["B12X_ROCE_HCA"] == "rocep1s0f0,rocep1s0f1,roceP2p1s0f0,roceP2p1s0f1" - assert env["B12X_ROCE_PEER_HCA_MAP"] == f"{1 - rank}=0/2" + # Only the selected functions are exposed to the RoCE proxy: it requires every + # listed device to be ACTIVE, and the f1 functions carry no cable on a + # single-DAC install. Peer-map indices address the rendered list. + assert env["B12X_ROCE_HCA"] == "rocep1s0f0,roceP2p1s0f0" + assert env["B12X_ROCE_PEER_HCA_MAP"] == f"{1 - rank}=0/1" assert env["NCCL_IB_HCA"] == "=rocep1s0f0,roceP2p1s0f0" assert env["B12X_ROCE_PAIR_PATHS"] == "2" assert env["NCCL_MIN_NCHANNELS"] == env["NCCL_MAX_NCHANNELS"] == "8" @@ -243,7 +246,7 @@ def test_tp2_clears_inherited_mesh_sitecustomize(inputs, tmp_path): @pytest.mark.parametrize( "assignment", [ - "B12X_ROCE_PEER_HCA_MAP=1=2/1", + "B12X_ROCE_PEER_HCA_MAP=1=1/0", "VLLM_PLUGINS=sparkcache", "SPARK_CACHE_ENABLED=1", "VLLM_HOST_IP=duplicate", From 6db2bfcb1901face86ac6a2829d5fcd280995c0b Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 13 Sep 2026 11:41:48 -0500 Subject: [PATCH 2/2] Document peer-map positions in the selected TP2 device list Distinguish physical inventory indices 0/2 from peer-map positions 0/1. No runtime behavior changes. TP2 and R33 profile tests pass. --- runtime/profiles/glm53-flash-spark-tp2/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/profiles/glm53-flash-spark-tp2/README.md b/runtime/profiles/glm53-flash-spark-tp2/README.md index 10997936..e424a3d1 100644 --- a/runtime/profiles/glm53-flash-spark-tp2/README.md +++ b/runtime/profiles/glm53-flash-spark-tp2/README.md @@ -144,7 +144,8 @@ transport, memory-guard, and cache settings come from the profile. Confirm this hardware inventory maps to the intended physical cage: `rocep1s0f0,rocep1s0f1,roceP2p1s0f0,roceP2p1s0f1`. The reciprocal peer maps -are rank 0 `1=0/2` and rank 1 `0=0/2`; both selected functions belong to p0. +use positions in the rendered two-device list: rank 0 `1=0/1` and rank 1 +`0=0/1`. These still select inventory functions 0 and 2, both belonging to p0. Install the host memory-guard service and apply [memory-guard.conf](memory-guard.conf) as its systemd drop-in. Both `create` and `start` require the active guard's effective 2 GiB floor. The launcher