From 9fd4f61d3d8c04939721c0674bd804b0b106bb5c Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:49:14 -0500 Subject: [PATCH] Add exact GLM-5.3 adaptive MTP and DFlash7 runtimes --- README.md | 5 +- ..._ADAPTIVE_MTP_SPARKCACHE_TP4_QUICKSTART.md | 17 +- ...YTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md | 124 ++++ ...GLM53_E10536A_SPARKCACHE_TP4_QUICKSTART.md | 10 +- docs/PREREQUISITES.md | 1 + recipes/sparkcache/README.md | 2 +- .../deepseek-v4-flash-0731-tp2-dcp1.json | 4 +- .../deepseek-v4-flash-0731-tp4-dcp1.json | 4 +- .../glm52-exl3-r7-3.5bpw-tp4-dcp4.json | 4 +- ...m53-flash-nvfp4-dflash2-bf16-tp4-dcp1.json | 4 +- runtime/README.md | 1 + .../Containerfile | 190 ++++++ .../README.md | 106 ++++ .../build-image.sh | 122 ++++ .../overlay_contract.py | 576 ++++++++++++++++++ .../010-dflash-draft-load-config.patch | 13 + .../pins.json | 132 ++++ .../prepare_context.py | 434 +++++++++++++ .../test_public_python_overlay.py | 295 +++++++++ .../verify_image.py | 354 +++++++++++ .../vllm-python-overlay.json | 232 +++++++ .../README.md | 6 +- .../pins.json | 5 +- ...st_glm53_b12x_kda_adaptive_mtp_contract.py | 6 +- .../README.md | 71 +++ .../build-image.sh | 127 ++++ .../pins.json | 172 ++++++ .../prepare_context.py | 271 ++++++++ .../remove_distribution.py | 125 ++++ .../test_dflash7_python_overlay.py | 274 +++++++++ .../verify_image.py | 48 ++ scripts/config/README.md | 19 +- ...fetensors-sparkcache-tp4-dcp1.example.json | 15 +- ...ash2-bf16-tp4-dcp1-sparkcache.example.json | 2 +- ...fetensors-sparkcache-tp4-dcp1.example.json | 201 ++++++ ...fetensors-sparkcache-tp4-dcp1.example.json | 198 ++++++ ...ash2-bf16-sparkcache-tp4-dcp1.example.json | 4 +- ...fetensors-sparkcache-tp4-dcp1.example.json | 4 +- ...-adaptive-sparkcache-tp4-dcp1.example.json | 4 +- ...536a-mtp5-sparkcache-tp4-dcp1.example.json | 4 +- ...fetensors-sparkcache-tp4-dcp1.example.json | 163 +++++ ...are_glm53_b12x_kda_adaptive_mtp_profile.py | 56 +- ...re_glm53_dflash7_python_overlay_profile.py | 352 +++++++++++ scripts/prepare_glm53_e105_profile.py | 49 +- ...are_glm53_public_python_overlay_profile.py | 324 ++++++++++ scripts/sparkcache_terminology.py | 151 +++++ scripts/sparkring_runtime.py | 9 + scripts/test_glm53_flash_profile.py | 3 +- ...are_glm53_b12x_kda_adaptive_mtp_profile.py | 84 ++- ...re_glm53_dflash7_python_overlay_profile.py | 271 ++++++++ scripts/test_prepare_glm53_e105_profile.py | 15 +- ...are_glm53_public_python_overlay_profile.py | 255 ++++++++ scripts/test_sparkcache_recipes.py | 3 +- scripts/test_sparkcache_terminology.py | 146 +++++ scripts/test_sparkring_generic_launcher.py | 44 ++ 55 files changed, 6028 insertions(+), 83 deletions(-) create mode 100644 docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md create mode 100644 runtime/glm53-flash-adaptive-mtp-python-overlay/Containerfile create mode 100644 runtime/glm53-flash-adaptive-mtp-python-overlay/README.md create mode 100644 runtime/glm53-flash-adaptive-mtp-python-overlay/build-image.sh create mode 100644 runtime/glm53-flash-adaptive-mtp-python-overlay/overlay_contract.py create mode 100644 runtime/glm53-flash-adaptive-mtp-python-overlay/patches/010-dflash-draft-load-config.patch create mode 100644 runtime/glm53-flash-adaptive-mtp-python-overlay/pins.json create mode 100644 runtime/glm53-flash-adaptive-mtp-python-overlay/prepare_context.py create mode 100644 runtime/glm53-flash-adaptive-mtp-python-overlay/test_public_python_overlay.py create mode 100644 runtime/glm53-flash-adaptive-mtp-python-overlay/verify_image.py create mode 100644 runtime/glm53-flash-adaptive-mtp-python-overlay/vllm-python-overlay.json create mode 100644 runtime/glm53-flash-dflash7-python-overlay/README.md create mode 100644 runtime/glm53-flash-dflash7-python-overlay/build-image.sh create mode 100644 runtime/glm53-flash-dflash7-python-overlay/pins.json create mode 100644 runtime/glm53-flash-dflash7-python-overlay/prepare_context.py create mode 100644 runtime/glm53-flash-dflash7-python-overlay/remove_distribution.py create mode 100644 runtime/glm53-flash-dflash7-python-overlay/test_dflash7_python_overlay.py create mode 100644 runtime/glm53-flash-dflash7-python-overlay/verify_image.py create mode 100644 scripts/config/glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json create mode 100644 scripts/config/glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json create mode 100644 scripts/config/glm53-flash-public-python-overlay-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json create mode 100644 scripts/prepare_glm53_dflash7_python_overlay_profile.py create mode 100644 scripts/prepare_glm53_public_python_overlay_profile.py create mode 100644 scripts/sparkcache_terminology.py create mode 100644 scripts/test_prepare_glm53_dflash7_python_overlay_profile.py create mode 100644 scripts/test_prepare_glm53_public_python_overlay_profile.py create mode 100644 scripts/test_sparkcache_terminology.py diff --git a/README.md b/README.md index 555d94f4..9f8ed1ec 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,9 @@ See the [profile registry](docs/profiles/README.md) for recipe identities and ev ### GLM-5.3 Flash research observation **Research-only — 16K context, single observation.** The SparkCache-enabled -profile recorded 2,371 tok/s prefill and 36.06 tok/s sustained -C1 decode on random tokens. No A/B baseline has been completed. +profile recorded 2,371 tok/s prefill and 36.06 tok/s sustained C1 decode on +random tokens. No A/B baseline has been completed. C4 and C8 were capacity-limited +and are omitted rather than reported as throughput results. | Profile | Prefill | C1 decode | C8 decode | Highest valid decode | Coding peak | |---|---:|---:|---:|---:|---:| diff --git a/docs/GLM53_B12X_KDA_ADAPTIVE_MTP_SPARKCACHE_TP4_QUICKSTART.md b/docs/GLM53_B12X_KDA_ADAPTIVE_MTP_SPARKCACHE_TP4_QUICKSTART.md index fe9496e6..a646bb7b 100644 --- a/docs/GLM53_B12X_KDA_ADAPTIVE_MTP_SPARKCACHE_TP4_QUICKSTART.md +++ b/docs/GLM53_B12X_KDA_ADAPTIVE_MTP_SPARKCACHE_TP4_QUICKSTART.md @@ -2,7 +2,8 @@ Status: **implemented, not qualified**. This guide builds vLLM commit `0b67266a0f37d6146a8403fb8482403c62f412d5` and the SparkCache overlay from -commit `20838ace3ebda570ca039cb7f1976c29da554b39` for four DGX Spark systems at +commit `5d571018de5b63a9a90e5c11e6d6e86bbff4a957`, Git tree +`e864ed9ad64f771188fdb59aa9738e348134d636`, for four DGX Spark systems at TP4/DCP1. The serving profile uses embedded MTP with maximum depth five, initial depth @@ -10,7 +11,7 @@ three, and a 32-step acceptance window. Fastsafetensors uses queue size one. TP4 makes the pinned vLLM loader select `nogds=True`, so model loading uses pipelined host I/O without GPU Direct Storage. -The profile reserves 20 GiB of FP8 KV per rank and enables SparkCache native +The profile reserves 20 GiB of FP8 KV per rank and enables SparkCache CUDA restore, tail-only publication, shared restore trunks, and bounded shared GPU prefix leases. Image construction and distribution do not require stopping an existing service. Do not run the launch command until all four ranks have the @@ -25,7 +26,7 @@ storage. Clone both repositories beside each other: git clone https://github.com/FujitsuPolycom/sparkring.git sparkring git -C sparkring checkout --detach git clone https://github.com/FujitsuPolycom/sparkcache.git sparkcache -git -C sparkcache checkout --detach 20838ace3ebda570ca039cb7f1976c29da554b39 +git -C sparkcache checkout --detach 5d571018de5b63a9a90e5c11e6d6e86bbff4a957 IMAGE='sparkring-glm53-runtime:b12x-kda-adaptive-mtp-0b67266a-arm64' \ BUILD_RECEIPT="$PWD/glm53-b12x-kda-adaptive-mtp-runtime-receipt.json" \ @@ -38,8 +39,8 @@ python sparkcache/deploy/glm53_flash/build_image.py \ --containerfile deploy/glm53_flash/Containerfile.b12x-kda-adaptive-mtp \ --base-image "${runtime_image}" \ --base-image-id "${runtime_image_id}" \ - --source-sha256 4998b24f4f504aeeb9bf92769ec720e282f546e6726d89fdfd06c4efa8d17c10 \ - --sparkcache-revision 20838ace3ebda570ca039cb7f1976c29da554b39 \ + --source-sha256 f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88 \ + --sparkcache-revision 5d571018de5b63a9a90e5c11e6d6e86bbff4a957 \ --output-image sparkring-glm53-sparkcache:b12x-kda-adaptive-mtp-0b67266a-arm64 ``` @@ -48,10 +49,10 @@ Record immutable local identities: ```bash sparkcache_image='sparkring-glm53-sparkcache:b12x-kda-adaptive-mtp-0b67266a-arm64' sparkcache_image_id="$(docker image inspect --format '{{.Id}}' "${sparkcache_image}")" -native_sha256="$(docker run --rm --entrypoint sha256sum "${sparkcache_image}" \ +cuda_placement_sha256="$(docker run --rm --entrypoint sha256sum "${sparkcache_image}" \ /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so \ | cut -d ' ' -f1)" -test "${#native_sha256}" -eq 64 +test "${#cuda_placement_sha256}" -eq 64 ``` The runtime builder verifies the complete first-parent vLLM history from @@ -71,7 +72,7 @@ python sparkring/scripts/prepare_glm53_b12x_kda_adaptive_mtp_profile.py \ --image-id "${sparkcache_image_id}" \ --parent-image "${runtime_image}" \ --parent-image-id "${runtime_image_id}" \ - --native-library-sha256 "${native_sha256}" \ + --cuda-placement-library-sha256 "${cuda_placement_sha256}" \ --profile-output profile.json \ --site-output site.yaml ``` diff --git a/docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md b/docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md new file mode 100644 index 00000000..390e4502 --- /dev/null +++ b/docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md @@ -0,0 +1,124 @@ +# Serve GLM-5.3 with external DFlash7 and the exact Python-overlay runtime + +Status: **implemented**, not qualified. The image builder, profile resolver, +and four-rank dry-run contract pass without GPUs. No image digest from this +path has completed TP4/DCP1 model loading, semantic generation, SparkCache +store/restart/restore, or concurrency qualification. + +## Runtime contract + +| Role | Exact identity | +|---|---| +| vLLM native extensions and wheel metadata | `da4d7be6c97434f6942292ed8abbf4b32dc44355` | +| vLLM Python source | `0b67266a0f37d6146a8403fb8482403c62f412d5`, tree `ba9484ccb33aa56e90ff2f447f15ca9b9da97639` | +| B12X | `b1d541f9e71a35f030d45fae437630fff7507c2a`, tree `c69cdec1c59a08e8e0e549f930fa8abcfb5134ae` | +| SparkCache reconstructed-page placement, canonical CUDA configuration, and bounded page-delta reads | `5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3`, tree `94c236b9dfbf5f70075eb47877fd9caaa5d8c249`, clean source SHA-256 `bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63` | +| DFlash draft-loader separation | Patch SHA-256 `39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279`, postimage SHA-256 `98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4` | +| Unused DeepEP removal | Distribution `deep_ep==2.0.0+local`, removal receipt SHA-256 `65514f44829e7d176b0b2cacc9559ed22724e525b7041a8bcd4d2e02d1f372e3` | +| Target | `local-inference-lab/GLM-5.3-Flash-NVFP4@520de24eabf507659eaef7c70f14fd584527facc` | +| External draft | `incoai/GLM-5.3-Flash-DFlash2@dc77ff1c99eeb2df044ee3d4f0094eb033fee410`, BF16 weights SHA-256 `b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b` | + +The serving contract uses seven speculative tokens, draft TP4, target FP8 KV, +32 sequences, and 256-token vLLM blocks. SparkCache selects +`tail-cow-v1`, which maps opaque GLM pages to the `page-tail-cow-v1` +namespace, and uses the canonical CUDA restore keys. + +## Build the image + +Run on Linux ARM64 from a clean checkout containing this guide: + +```bash +IMAGE='sparkring-glm53-sparkcache:dflash7-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64' \ +BUILD_RECEIPT="$PWD/glm53-dflash7-python-overlay-image-receipt.json" \ +bash runtime/glm53-flash-dflash7-python-overlay/build-image.sh +``` + +The builder verifies the public da4 image, the 31-file Python overlay, retained +native ELFs and dispatch operators, B12X, SparkCache clean source, the CUDA +placement library, four exact vLLM patches, and the eleven-file lease contract. +The base-image inspection must identify exactly one installed distribution, +`deep_ep==2.0.0+local`, as the owner of the `deep_ep` module. The derived image +uninstalls that exact distribution and verifies that `deep_ep` is absent. The +profiles use B12X kernels and PYNCCL collectives, so DeepEP is not a serving +dependency. The builder does not push the image. + +Both profiles leave Torch thread selection to vLLM, select language-model-only +serving, and disable the unsupported symmetric-memory and FlashInfer all-reduce +candidates. PYNCCL remains bound to `/opt/sparkring/nccl/libnccl.so.2`, B12X +remains the attention, MoE, and linear backend, and the all-reduce RMS fusion is +disabled. ModelOpt experimental-quantization and FP8 KV accuracy warnings remain +visible because they describe real format and accuracy limitations; the profiles +do not suppress warnings. + +## Choose the target loader + +Two profiles share the same image and DFlash7 cache identity: + +| Profile | Status | Loader behavior | +|---|---|---| +| `glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json` | **implemented**, not qualified | Uses global safetensors for target and draft. This follows the qualified-compatible loader shape but still requires live qualification on the composed 0b image. | +| `glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json` | **implemented**, not qualified | Uses global fastsafetensors with queue size one for the target and `draft_load_config={"load_format":"safetensors"}` for DFlash. | + +The image applies an exact-input vLLM patch that passes +`SpeculativeConfig.draft_load_config` to the DFlash model loader. The image +receipt verifies patch SHA-256 +`39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279` and +postimage SHA-256 +`98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4`. +Both profiles remain unqualified until live four-rank gates pass. + +SparkCache PR #26 accepts the canonical CUDA keys used by both profiles. No +local PR25 compatibility profile or legacy-key rewrite is part of this path. + +## Resolve the profile and inspect the plan + +Copy `scripts/config/glm53-flash-tp4-site.example.yaml` +outside version control and replace every address, interface, SSH target, +device, host path, and image identity. Select one profile template: + +```bash +receipt="$PWD/glm53-dflash7-python-overlay-image-receipt.json" +image='sparkring-glm53-sparkcache:dflash7-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64' +profile_template='scripts/config/glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json' + +python scripts/prepare_glm53_dflash7_python_overlay_profile.py \ + --profile-template "$profile_template" \ + --site-template /path/to/resolved-glm53-site.yaml \ + --image "$image" \ + --image-id "$(jq -r .image_id "$receipt")" \ + --cuda-placement-library-sha256 "$(jq -r .artifacts.sparkcache_cuda_placement_sha256 "$receipt")" \ + --native-elf-manifest-sha256 "$(jq -r .runtime_contract.native_elf_manifest_sha256 "$receipt")" \ + --native-dispatch-manifest-sha256 "$(jq -r .runtime_contract.native_dispatch_manifest_sha256 "$receipt")" \ + --source-receipt-sha256 "$(jq -r .artifacts.source_receipt_sha256 "$receipt")" \ + --profile-output /path/to/glm53-dflash7-profile.json \ + --site-output /path/to/glm53-dflash7-site.yaml + +python scripts/sparkring_generic_launcher.py \ + --site /path/to/glm53-dflash7-site.yaml \ + --profile /path/to/glm53-dflash7-profile.json \ + plan +``` + +`plan` is offline. Inspect every rank action before a lifecycle command. + +## Cache namespace impact + +The external DFlash weights SHA-256 is stored as +`spark_cache_draft_checkpoint_sha256`. It cannot share entries with embedded +MTP profiles. `tail-cow-v1` also separates these entries from snapshot-v1 +manifests. The two target-loader profiles share a namespace because loader +choice does not change target or draft model state; each profile uses a +different cache root and one-shot clear token while qualification is pending. + +[SparkCache pull request #30](https://github.com/FujitsuPolycom/sparkcache/pull/30) +combines canonical CUDA configuration names, replacement of a partial terminal +HMA page when an authenticated cache boundary falls inside that page, and an +eight-worker reader for authenticated page-delta chunks. The reader preserves +manifest descriptor order after concurrent reads. +Moving from SparkCache commit +`5d571018de5b63a9a90e5c11e6d6e86bbff4a957` to the pinned commit +`5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3` does not change the namespace. +Checkpoint identities, page-delta wire schemas, record vocabulary, digest +salts, parallel geometry, vLLM patches, the lease contract, and the CUDA +placement ABI are unchanged. Compatible `page-tail-cow-v1` entries remain +eligible. Null-block publication failures remain unsupported. diff --git a/docs/GLM53_E10536A_SPARKCACHE_TP4_QUICKSTART.md b/docs/GLM53_E10536A_SPARKCACHE_TP4_QUICKSTART.md index 6c0cda3c..829fbed2 100644 --- a/docs/GLM53_E10536A_SPARKCACHE_TP4_QUICKSTART.md +++ b/docs/GLM53_E10536A_SPARKCACHE_TP4_QUICKSTART.md @@ -19,8 +19,8 @@ pipelined shard loading without GPU Direct Storage. The queue can retain one additional shard-sized device buffer during model loading. All profiles use TP4/DCP1, 20 GiB FP8 KV per rank, a 524,288-token request -limit, 32 sequences, native direct restore, two restore lanes, eight native -I/O workers, two 256 MiB arenas, shared GPU-prefix leases, and one-shot cache +limit, 32 sequences, SparkCache CUDA restore, two restore lanes, eight CUDA +restore I/O workers, two 256 MiB placement arenas, shared GPU-prefix leases, and one-shot cache clearing. ## Build immutable inputs @@ -57,10 +57,10 @@ Record the exact output identities: ```bash sparkcache_image='sparkring-glm53-sparkcache:e10536a-source-arm64' sparkcache_image_id="$(docker image inspect --format '{{.Id}}' "${sparkcache_image}")" -native_sha256="$(docker run --rm --entrypoint sha256sum "${sparkcache_image}" \ +cuda_placement_sha256="$(docker run --rm --entrypoint sha256sum "${sparkcache_image}" \ /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so \ | cut -d ' ' -f1)" -test "${#native_sha256}" -eq 64 +test "${#cuda_placement_sha256}" -eq 64 ``` ## Resolve a serving profile @@ -88,7 +88,7 @@ python sparkring/scripts/prepare_glm53_e105_profile.py \ --image-id "${sparkcache_image_id}" \ --parent-image "${runtime_image}" \ --parent-image-id "${runtime_image_id}" \ - --native-library-sha256 "${native_sha256}" \ + --cuda-placement-library-sha256 "${cuda_placement_sha256}" \ --profile-output profile.json \ --site-output site.yaml ``` diff --git a/docs/PREREQUISITES.md b/docs/PREREQUISITES.md index ebedd273..b49e57ce 100644 --- a/docs/PREREQUISITES.md +++ b/docs/PREREQUISITES.md @@ -6,6 +6,7 @@ hardware and operator conditions required by the [GLM-5.3 Flash SparkCache quickstart](GLM53_FLASH_DFLASH2_BF16_SPARKCACHE_TP4_QUICKSTART.md), [source-built GLM-5.3 e10536a quickstart](GLM53_E10536A_SPARKCACHE_TP4_QUICKSTART.md), [GLM-5.3 adaptive-MTP and live-tensor KDA quickstart](GLM53_B12X_KDA_ADAPTIVE_MTP_SPARKCACHE_TP4_QUICKSTART.md), +[GLM-5.3 DFlash7 Python-overlay quickstart](GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md), [GLM-5.3 Flash cache-disabled quickstart](GLM53_FLASH_DFLASH2_BF16_TP4_QUICKSTART.md), [DeepSeek quickstart](DEEPSEEK_V4_FLASH_QUICKSTART.md), [Qwen3.8-27B pair quickstart](QWEN38_27B_EXL3_K5K6_PAIR_QUICKSTART.md), and diff --git a/recipes/sparkcache/README.md b/recipes/sparkcache/README.md index e5d8c2ab..b2f9690f 100644 --- a/recipes/sparkcache/README.md +++ b/recipes/sparkcache/README.md @@ -90,7 +90,7 @@ Limitations: The published receipts use a 4,096-token scheduler budget; other budgets are operator choices whose performance and capacity behavior is not recorded here, except that the GLM-5.3 Flash receipt uses 8,192. The DeepSeek receipts cover DCP1, not DCP2 or DCP4. These -recipes disable streaming snapshots and native restore, so the receipts do not +recipes disable streaming snapshots and SparkCache CUDA restore, so the receipts do not cover either mode. Other images, checkpoints, and cache geometries are also outside the recorded evidence. The GLM-5.3 Flash receipt covers an 8,192-token restored span and does not establish throughput neutrality or larger-span diff --git a/recipes/sparkcache/deepseek-v4-flash-0731-tp2-dcp1.json b/recipes/sparkcache/deepseek-v4-flash-0731-tp2-dcp1.json index 2498ac72..247fa51c 100644 --- a/recipes/sparkcache/deepseek-v4-flash-0731-tp2-dcp1.json +++ b/recipes/sparkcache/deepseek-v4-flash-0731-tp2-dcp1.json @@ -64,7 +64,7 @@ "min_span_tokens": 256, "max_span_tokens": 1048576, "streaming_snapshots": false, - "native_restore": false, + "cuda_restore": false, "root_requirement": "Use a dedicated rank-local host directory mounted at the same container path on both ranks." }, "evidence": { @@ -78,7 +78,7 @@ "The 17179869184-byte reservation, block size 256, and 4096-token scheduler budget carry forward unchanged, but their capacity with the larger context and sequence target must be measured.", "The complete checkpoint manifest is generated from the pinned revision and must reproduce the recorded digest. A different digest is a different composition and does not reproduce this qualification.", "The published receipt covers DeepSeek DCP1, not DCP2 or DCP4.", - "This recipe disables streaming snapshots and native restore; the published receipt does not cover either mode." + "This recipe disables streaming snapshots and SparkCache CUDA restore; the published receipt does not cover either mode." ], "record": "recipes/sparkcache/README.md", "artifact_scope": "The exact SparkCache wheel, runtime image, checkpoint identity, topology, and historical max_model_len=131072, max_num_seqs=6 serving values stated in evidence.conditions.", diff --git a/recipes/sparkcache/deepseek-v4-flash-0731-tp4-dcp1.json b/recipes/sparkcache/deepseek-v4-flash-0731-tp4-dcp1.json index 0f0c1396..bde56c60 100644 --- a/recipes/sparkcache/deepseek-v4-flash-0731-tp4-dcp1.json +++ b/recipes/sparkcache/deepseek-v4-flash-0731-tp4-dcp1.json @@ -54,7 +54,7 @@ "min_span_tokens": 256, "max_span_tokens": 1048576, "streaming_snapshots": false, - "native_restore": false, + "cuda_restore": false, "root_requirement": "Use one dedicated rank-local host directory per physical rank, mounted at the same container path on all four ranks." }, "evidence": { @@ -68,7 +68,7 @@ "The 34359738368-byte reservation, 32-sequence limit, block size 256, and 4096-token scheduler budget carry forward unchanged, but capacity at the larger context must be measured.", "The complete checkpoint manifest is generated from the pinned revision and must reproduce the recorded digest. A different digest is a different composition and does not reproduce this qualification.", "The published receipt covers DeepSeek DCP1, not DCP2 or DCP4.", - "This recipe disables streaming snapshots and native restore; the published receipt does not cover either mode." + "This recipe disables streaming snapshots and SparkCache CUDA restore; the published receipt does not cover either mode." ], "record": "recipes/sparkcache/README.md", "artifact_scope": "The exact SparkCache wheel, runtime image, checkpoint identity, topology, and historical max_model_len=524288 serving values stated in evidence.conditions.", diff --git a/recipes/sparkcache/glm52-exl3-r7-3.5bpw-tp4-dcp4.json b/recipes/sparkcache/glm52-exl3-r7-3.5bpw-tp4-dcp4.json index f4ad30c8..10050b5c 100644 --- a/recipes/sparkcache/glm52-exl3-r7-3.5bpw-tp4-dcp4.json +++ b/recipes/sparkcache/glm52-exl3-r7-3.5bpw-tp4-dcp4.json @@ -57,7 +57,7 @@ "min_span_tokens": 256, "max_span_tokens": 1048576, "streaming_snapshots": false, - "native_restore": false, + "cuda_restore": false, "root_requirement": "Use one dedicated rank-local host directory per physical rank, mounted at the same container path on all four ranks." }, "restart_contract": { @@ -75,7 +75,7 @@ "The qualified receipt covers max_model_len=262144; the current max_model_len=1048576 target requires a larger CKV-gather workspace and live memory validation.", "The qualified operator image is identified by local image ID but has no published immutable image reference. A clean-checkout rebuild has implemented status until it passes the promotion checklist.", "The qualified lane requires preserving and regenerating the 40-query-row exact-state receipt, whose path contains q40-exact-state-serving-v1, before every coordinated restart.", - "This recipe disables streaming snapshots and native restore; the published receipt does not cover either mode.", + "This recipe disables streaming snapshots and SparkCache CUDA restore; the published receipt does not cover either mode.", "Full GLM reasoning-trace equality is an inconclusive diagnostic under this runtime: repeated fixed-seed requests produced different non-empty reasoning bodies with identical final content and finish reason stop." ], "record": "https://github.com/FujitsuPolycom/sparkcache/blob/999bc13d6c0b52a6cee3c90487ac9b8643ba3f99/GLM52_A2_LIVE_VALIDATION.md", diff --git a/recipes/sparkcache/glm53-flash-nvfp4-dflash2-bf16-tp4-dcp1.json b/recipes/sparkcache/glm53-flash-nvfp4-dflash2-bf16-tp4-dcp1.json index 7b0a3d75..cb43e1fe 100644 --- a/recipes/sparkcache/glm53-flash-nvfp4-dflash2-bf16-tp4-dcp1.json +++ b/recipes/sparkcache/glm53-flash-nvfp4-dflash2-bf16-tp4-dcp1.json @@ -83,7 +83,7 @@ "min_span_tokens": 4096, "max_span_tokens": 524288, "streaming_snapshots": false, - "native_restore": false, + "cuda_restore": false, "root_requirement": "Use one dedicated rank-local host directory per physical rank, mounted at /cache/jit on every rank." }, "evidence": { @@ -97,7 +97,7 @@ "A rebuilt image has implemented status until its immutable digest passes the same live qualification.", "The first request after API startup recomputed because the scheduler had not received a complete four-rank inventory checkpoint; the following identical request formed quorum and restored.", "The receipt does not establish throughput neutrality or restore performance for spans larger than 8192 tokens.", - "Streaming snapshots, native direct restore, MTP drafting, and other DFlash checkpoints are unsupported by this evidence.", + "Streaming snapshots, SparkCache CUDA restore, MTP drafting, and other DFlash checkpoints are unsupported by this evidence.", "The source-built runtime uses the Triton KDA prefill backend and stock safetensors loading; FlashKDA prefill and InstantTensor checkpoint loading are unsupported by this image.", "The optional deep_ep import emits a duplicate-NCCL warning. vLLM selects the source-built NCCL library through VLLM_NCCL_SO_PATH and serving proceeds.", "The target repository does not record its base-checkpoint revision." diff --git a/runtime/README.md b/runtime/README.md index f9b9a7fa..b46f54fb 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -13,6 +13,7 @@ credentials, or a live-deployment result. | [`glm53-flash/`](glm53-flash/README.md) | GLM-5.3 Flash target, BF16 DFlash2, vLLM, B12X, patched NCCL, and SparkCache identity and attestation contract | | [`glm53-flash-e10536a/`](glm53-flash-e10536a/README.md) | Implemented source builder for vLLM e10536a with internal MTP5 and opt-in adaptive depth; live serving unqualified | | [`glm53-flash-b12x-kda-adaptive-mtp/`](glm53-flash-b12x-kda-adaptive-mtp/README.md) | Implemented source builder for adaptive MTP and live-tensor B12X KDA at vLLM `0b67266a`; live serving unqualified | +| [`glm53-flash-dflash7-python-overlay/`](glm53-flash-dflash7-python-overlay/README.md) | Implemented public-base builder for external DFlash7 with vLLM `0b67266a` Python over retained `da4d7be` native extensions; live serving unqualified | | [`deepseek0731-gb10/`](deepseek0731-gb10/README.md) | DeepSeek-V4-Flash-0731 GB10 parser, K5 sparse-row, and native PR431 image layer | | [`qwen38/`](qwen38/README.md) | Public-source ARM64 image builder for the Qwen3.8-27B EXL3 K5/K6 pair and cycle profiles | | [`faststart-lock.json`](faststart-lock.json) | Immutable ARM64 base-image and model-identity pins | diff --git a/runtime/glm53-flash-adaptive-mtp-python-overlay/Containerfile b/runtime/glm53-flash-adaptive-mtp-python-overlay/Containerfile new file mode 100644 index 00000000..3b43225e --- /dev/null +++ b/runtime/glm53-flash-adaptive-mtp-python-overlay/Containerfile @@ -0,0 +1,190 @@ +# syntax=docker/dockerfile:1.7 + +ARG PUBLIC_BASE=ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd +ARG ARM_BUILDER=pytorch/manylinuxaarch64-builder@sha256:f91599c49f526c77d01b68286f2bf943a5fd6a432d7e3f0afcc5784825908fe9 + +FROM ${ARM_BUILDER} AS b12x-wheel +ARG B12X_COMMIT +ARG B12X_TREE +COPY bundle/sources/b12x /build/b12x +RUN test "$(git -C /build/b12x rev-parse HEAD)" = "${B12X_COMMIT}" \ + && test "$(git -C /build/b12x rev-parse HEAD^{tree})" = "${B12X_TREE}" \ + && /opt/python/cp312-cp312/bin/python -m pip wheel \ + --no-deps --wheel-dir /out/wheels /build/b12x \ + && sha256sum /out/wheels/b12x-*.whl > /out/wheels/b12x-wheel.sha256 + +FROM ${ARM_BUILDER} AS sparkcache-cuda-placement +COPY bundle/sources/sparkcache/sparkcache/native /build/sparkcache-cuda-placement +RUN cmake -S /build/sparkcache-cuda-placement \ + -B /build/sparkcache-cuda-placement/build-cuda \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_ARCHITECTURES=121 \ + -DSPARK_CACHE_PLACEMENT_ENABLE_CUDA=ON \ + && cmake --build /build/sparkcache-cuda-placement/build-cuda \ + --target spark_cache_placement -j"$(nproc)" \ + && test -s /build/sparkcache-cuda-placement/build-cuda/libspark_cache_placement.so \ + && sha256sum /build/sparkcache-cuda-placement/build-cuda/libspark_cache_placement.so \ + > /build/sparkcache-cuda-placement/build-cuda/libspark_cache_placement.so.sha256 + +FROM ${PUBLIC_BASE} + +ARG PUBLIC_BASE +ARG PUBLIC_BASE_ID +ARG VLLM_NATIVE_COMMIT +ARG VLLM_PYTHON_COMMIT +ARG VLLM_PYTHON_TREE +ARG B12X_COMMIT +ARG B12X_TREE +ARG SPARKCACHE_COMMIT +ARG SPARKCACHE_TREE +ARG SPARKCACHE_SOURCE_SHA256 +ARG SPARKRING_REVISION +ARG SOURCE_RECEIPT_SHA256 +ARG OVERLAY_MANIFEST_SHA256 +ARG NATIVE_ELF_MANIFEST_SHA256 +ARG NATIVE_DISPATCH_MANIFEST_SHA256 +ARG SPARKCACHE_CUDA_PLACEMENT_SHA256 + +ENV PYTHON_OVERLAY_ROOT=/opt/sparkring/runtime/python-overlay \ + VLLM_CACHE_ROOT=/cache/jit/vllm/py-0b67266-native-da4d7be \ + B12X_CUTE_COMPILE_CACHE_DIR=/cache/jit/b12x/b1d541f9/cute \ + TRITON_CACHE_DIR=/cache/jit/triton/vllm-py-0b67266-native-da4d7be + +COPY bundle/runtime/pins.json ${PYTHON_OVERLAY_ROOT}/pins.json +COPY bundle/runtime/vllm-python-overlay.json ${PYTHON_OVERLAY_ROOT}/vllm-python-overlay.json +COPY bundle/runtime/overlay_contract.py ${PYTHON_OVERLAY_ROOT}/overlay_contract.py +COPY bundle/runtime/verify_image.py ${PYTHON_OVERLAY_ROOT}/verify_image.py +COPY receipt.json ${PYTHON_OVERLAY_ROOT}/source-receipt.json + +RUN set -eu; \ + test "$(sha256sum "${PYTHON_OVERLAY_ROOT}/vllm-python-overlay.json" | cut -d' ' -f1)" = \ + "${OVERLAY_MANIFEST_SHA256}"; \ + python3 "${PYTHON_OVERLAY_ROOT}/overlay_contract.py" \ + --pins "${PYTHON_OVERLAY_ROOT}/pins.json" \ + --manifest "${PYTHON_OVERLAY_ROOT}/vllm-python-overlay.json" \ + verify-files \ + --root /usr/local/lib/python3.12/dist-packages \ + --stage base; \ + python3 "${PYTHON_OVERLAY_ROOT}/overlay_contract.py" \ + --pins "${PYTHON_OVERLAY_ROOT}/pins.json" \ + --manifest "${PYTHON_OVERLAY_ROOT}/vllm-python-overlay.json" \ + record-base \ + --site-root /usr/local/lib/python3.12/dist-packages \ + --console-script /usr/local/bin/vllm \ + --output "${PYTHON_OVERLAY_ROOT}/retained-native.json"; \ + python3 -c 'import json,sys; record=json.load(open(sys.argv[1], encoding="utf-8")); assert record["native_elf_manifest_sha256"] == sys.argv[2]; assert record["native_dispatch_manifest_sha256"] == sys.argv[3]' \ + "${PYTHON_OVERLAY_ROOT}/retained-native.json" \ + "${NATIVE_ELF_MANIFEST_SHA256}" "${NATIVE_DISPATCH_MANIFEST_SHA256}" + +COPY bundle/vllm-overlay/ /usr/local/lib/python3.12/dist-packages/ +RUN python3 "${PYTHON_OVERLAY_ROOT}/overlay_contract.py" \ + --pins "${PYTHON_OVERLAY_ROOT}/pins.json" \ + --manifest "${PYTHON_OVERLAY_ROOT}/vllm-python-overlay.json" \ + verify-files \ + --root /usr/local/lib/python3.12/dist-packages \ + --stage target +COPY bundle/runtime/patches/010-dflash-draft-load-config.patch \ + ${PYTHON_OVERLAY_ROOT}/patches/010-dflash-draft-load-config.patch +RUN root=/usr/local/lib/python3.12/dist-packages; \ + target="${root}/vllm/v1/worker/gpu/spec_decode/dflash/utils.py"; \ + before="$(sha256sum "${target}" | cut -d' ' -f1)"; \ + printf 'dflash_loader_preimage_sha256=%s\n' "${before}"; \ + test "${before}" = 2301c8199b73ed893dfbd3ae14ad125816f100b2d2ed034215b1f2d9aa2c23c5; \ + patch --batch --forward -p1 -d "${root}" \ + < "${PYTHON_OVERLAY_ROOT}/patches/010-dflash-draft-load-config.patch"; \ + test "$(sha256sum "${target}" | cut -d' ' -f1)" = \ + 98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4 + +COPY --from=b12x-wheel /out/wheels/ /opt/sparkring/wheelhouse/ +RUN uv pip install --system --reinstall --no-deps \ + /opt/sparkring/wheelhouse/b12x-*.whl \ + && cp /opt/sparkring/wheelhouse/b12x-wheel.sha256 \ + "${PYTHON_OVERLAY_ROOT}/b12x-wheel.sha256" + +COPY bundle/sources/sparkcache/sparkcache /opt/sparkcache-src/sparkcache +COPY bundle/sources/sparkcache/deploy/deployment_contract/source.py \ + /opt/sparkcache-source-identity.py +RUN actual_source="$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location("source_identity", "/opt/sparkcache-source-identity.py"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path("/opt/sparkcache-src/sparkcache")))')" \ + && printf 'sparkcache_source_sha256=%s\n' "${actual_source}" \ + && test "${actual_source}" = "${SPARKCACHE_SOURCE_SHA256}" \ + && printf '%s\n' "${actual_source}" \ + > "${PYTHON_OVERLAY_ROOT}/sparkcache-source-tree.sha256" +COPY --from=sparkcache-cuda-placement \ + /build/sparkcache-cuda-placement/build-cuda/libspark_cache_placement.so \ + /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so +COPY --from=sparkcache-cuda-placement \ + /build/sparkcache-cuda-placement/build-cuda/libspark_cache_placement.so.sha256 \ + /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so.sha256 +COPY bundle/sources/sparkcache/patches/vllm-glm53-b12x-kda-adaptive-mtp \ + /opt/sparkcache-vllm-patches +COPY bundle/runtime/SparkRing-LICENSE /usr/share/licenses/SparkRing/LICENSE +COPY bundle/sources/vllm/LICENSE /usr/share/licenses/vLLM/LICENSE +COPY bundle/sources/b12x/LICENSE /usr/share/licenses/B12X/LICENSE +COPY bundle/sources/sparkcache/LICENSE /usr/share/licenses/SparkCache/LICENSE + +RUN set -eu; \ + actual_cuda_placement="$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d' ' -f1)"; \ + printf 'sparkcache_cuda_placement_sha256=%s\n' "${actual_cuda_placement}"; \ + test "${actual_cuda_placement}" = "${SPARKCACHE_CUDA_PLACEMENT_SHA256}"; \ + root=/usr/local/lib/python3.12/dist-packages; \ + config="${root}/vllm/config/vllm.py"; \ + config_sha256="$(sha256sum "${config}" | cut -d' ' -f1)"; \ + printf 'vllm_config_preimage_sha256=%s\n' "${config_sha256}"; \ + test "${config_sha256}" = cc03756d9bebf2a128828e5fbf7e9766446884dfce016c34381bce0aa78bfd9e; \ + patch --batch --forward -p1 -d "${root}" \ + < /opt/sparkcache-vllm-patches/020-sparkcache-vmm-exemption.patch; \ + test "$(sha256sum "${config}" | cut -d' ' -f1)" = 9f64f5041f7f9d953e9f6bc53de8733b3eb4035c0753056a1f646346702a0994; \ + scheduler="${root}/vllm/v1/core/sched/scheduler.py"; \ + test "$(sha256sum "${scheduler}" | cut -d' ' -f1)" = 05c05f4b372c7a4bf76399b38b338eab657c69406b7019ab02101d2ab0c7764c; \ + patch --batch --forward -p1 -d "${root}" \ + < /opt/sparkcache-vllm-patches/030-sparkcache-hma-load-failure.patch; \ + test "$(sha256sum "${scheduler}" | cut -d' ' -f1)" = 337893b6b088d12eb38d8d70c866242d085134442289b4f4574a1f162c9f11c8; \ + kv_manager="${root}/vllm/v1/core/kv_cache_manager.py"; \ + test "$(sha256sum "${kv_manager}" | cut -d' ' -f1)" = 02c71da26bbac81629248ee42b0a71bd2db817d339894d647d2ff6b66fd5ad19; \ + patch --batch --forward -p1 -d "${root}" \ + < /opt/sparkcache-vllm-patches/040-sparkcache-shared-prefix-lease.patch; \ + test "$(sha256sum "${kv_manager}" | cut -d' ' -f1)" = ee03dc9ce2b720c0be6e9f572d23580ba96eff68fe3406250557e83071654af0; \ + patch --batch --forward -p1 -d "${root}" \ + < /opt/sparkcache-vllm-patches/041-sparkcache-shared-prefix-attach.patch; \ + test "$(sha256sum "${scheduler}" | cut -d' ' -f1)" = 6d397c97f31e67a75efc01b5ddd89fa58db425de14fa43965ef2d6146b6b9bdb; \ + python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py \ + --vllm-root "${root}" \ + --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json; \ + python3 "${PYTHON_OVERLAY_ROOT}/overlay_contract.py" \ + --pins "${PYTHON_OVERLAY_ROOT}/pins.json" \ + --manifest "${PYTHON_OVERLAY_ROOT}/vllm-python-overlay.json" \ + verify-composed \ + --root "${root}" \ + --site-root "${root}" \ + --console-script /usr/local/bin/vllm \ + --base-record "${PYTHON_OVERLAY_ROOT}/retained-native.json" \ + --output "${PYTHON_OVERLAY_ROOT}/composed-verification.json"; \ + find "${root}/vllm" -type d -name __pycache__ -prune -exec rm -rf '{}' + + +ENV PYTHONPATH=/opt/sparkcache-src +ENTRYPOINT ["vllm"] + +LABEL org.opencontainers.image.title="SparkRing GLM-5.3 adaptive-MTP Python overlay" \ + org.opencontainers.image.description="vLLM 0b67266 Python over da4d7be native extensions with B12X b1d541f and SparkCache" \ + org.opencontainers.image.source="https://github.com/FujitsuPolycom/sparkring" \ + org.opencontainers.image.revision="${SPARKRING_REVISION}" \ + org.opencontainers.image.base.name="${PUBLIC_BASE}" \ + org.opencontainers.image.licenses="LicenseRef-NVIDIA-Deep-Learning-Container AND Apache-2.0 AND BSD-3-Clause" \ + org.sparkring.base.image-id="${PUBLIC_BASE_ID}" \ + org.sparkring.vllm.native.commit="${VLLM_NATIVE_COMMIT}" \ + org.sparkring.vllm.python.commit="${VLLM_PYTHON_COMMIT}" \ + org.sparkring.vllm.python.tree="${VLLM_PYTHON_TREE}" \ + org.sparkring.vllm.python-overlay-manifest-sha256="${OVERLAY_MANIFEST_SHA256}" \ + org.sparkring.vllm.dflash-draft-loader-patch-sha256="39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279" \ + org.sparkring.vllm.dflash-draft-loader-postimage-sha256="98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4" \ + org.sparkring.vllm.native-elf-manifest-sha256="${NATIVE_ELF_MANIFEST_SHA256}" \ + org.sparkring.vllm.native-dispatch-manifest-sha256="${NATIVE_DISPATCH_MANIFEST_SHA256}" \ + org.jovian.b12x.commit="${B12X_COMMIT}" \ + org.sparkring.b12x.tree="${B12X_TREE}" \ + org.sparkcache.source-revision="${SPARKCACHE_COMMIT}" \ + org.sparkcache.source-tree="${SPARKCACHE_TREE}" \ + org.sparkcache.source-sha256="${SPARKCACHE_SOURCE_SHA256}" \ + org.sparkcache.cuda-placement-library-sha256="${SPARKCACHE_CUDA_PLACEMENT_SHA256}" \ + org.sparkcache.vllm-contract-sha256="6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024" \ + org.sparkcache.deployment-profile="glm53-flash-adaptive-mtp-python-overlay" \ + org.sparkring.source-receipt-sha256="${SOURCE_RECEIPT_SHA256}" diff --git a/runtime/glm53-flash-adaptive-mtp-python-overlay/README.md b/runtime/glm53-flash-adaptive-mtp-python-overlay/README.md new file mode 100644 index 00000000..cbb9952b --- /dev/null +++ b/runtime/glm53-flash-adaptive-mtp-python-overlay/README.md @@ -0,0 +1,106 @@ +# GLM-5.3 public-base Python overlay + +Status: **implemented** for image construction and offline verification. The +image is unsupported for serving until an immutable digest passes four-rank +TP4/DCP1 model loading, semantic generation, SparkCache restore, +failure-recovery, and concurrency checks. + +This builder retains the compiled vLLM extensions and wheel metadata from the +qualified public runtime +`ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd`. +It replaces the 31 Python files changed between vLLM +`da4d7be6c97434f6942292ed8abbf4b32dc44355` and +`0b67266a0f37d6146a8403fb8482403c62f412d5`. + +The image also installs B12X +`b1d541f9e71a35f030d45fae437630fff7507c2a`. That revision provides the +trusted-metadata and request-sized live-tensor KDA interface required by the +vLLM Python sources. B12X +`2fcf23a0ce269be27b2e03fece73d46e90e6aeea` does not provide that interface +and is rejected. + +The builder records every retained vLLM ELF file, the native dispatch operator +set, generated version metadata, and wheel metadata before replacing Python +files. Construction fails if any retained artifact changes. The output labels +identify the retained native commit and overlaid Python commit separately; the +image is not described as a source-built vLLM 0b67266 wheel. + +The SparkCache source that routes reconstructed opaque pages through the SM121 +placement library is commit +`5d571018de5b63a9a90e5c11e6d6e86bbff4a957`, Git tree +`e864ed9ad64f771188fdb59aa9738e348134d636`. The builder verifies clean +deployable-source SHA-256 +`f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88` +before generating the SparkCache CUDA placement library. It applies the VMM exemption, +load-failure recovery, shared-prefix retention, and follower-attachment +patches in order, then runs the eleven-file lease-contract verifier. The test +profile selects `spark_cache_publication_schema=tail-cow-v1`, which maps opaque +GLM page storage to the distinct `page-tail-cow-v1` cache namespace. + +## Build + +Run on a Linux ARM64 host with Docker BuildKit: + +```bash +cd /path/to/sparkring +IMAGE='sparkring-glm53-sparkcache:vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64' \ +BUILD_RECEIPT="$PWD/glm53-public-python-overlay-image-receipt.json" \ +bash runtime/glm53-flash-adaptive-mtp-python-overlay/build-image.sh +``` + +The script pulls and verifies the immutable public base, fetches exact source +commits into a temporary build context, builds a pure B12X wheel and the +SparkCache CUDA placement library, creates the composed image, and writes a local +receipt. It does not push an image or contact serving hosts. + +## External DFlash loader separation + +Status: **implemented, not qualified**. The composed vLLM runtime passes +`SpeculativeConfig.draft_load_config` to the DFlash model loader. An external +BF16 DFlash2 checkpoint can therefore retain the standard safetensors loader +while the NVFP4 target uses fastsafetensors: + +```text +--load-format fastsafetensors +--speculative-config '{"method":"dflash","model":"/mtp-draft","num_speculative_tokens":7,"draft_tensor_parallel_size":4,"kv_cache_dtype":"auto","draft_sample_method":"probabilistic","rejection_sample_method":"standard","draft_load_config":{"load_format":"safetensors"}}' +``` + +Mount the exact external checkpoint read-only at `/mtp-draft`. The draft +loader config is optional: omitting it inherits the target `LoadConfig`, which +means both models use fastsafetensors under the command above. The separated +configuration requires `fastsafetensors==0.3.3` for the target and no +InstantTensor mount or environment setting. Four-rank model loading, +generation, draft counters, and peak device memory remain unqualified. + +## Resolve and inspect the four-rank plan + +Copy the sanitized site template outside version control and replace its +addresses, SSH targets, interfaces, devices, paths, and image identity. Resolve +the runtime profile from values in the image receipt: + +```bash +receipt="$PWD/glm53-public-python-overlay-image-receipt.json" +image='sparkring-glm53-sparkcache:vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64' + +python scripts/prepare_glm53_public_python_overlay_profile.py \ + --profile-template scripts/config/glm53-flash-public-python-overlay-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json \ + --site-template /path/to/resolved-glm53-site.yaml \ + --image "$image" \ + --image-id "$(jq -r .image_id "$receipt")" \ + --cuda-placement-library-sha256 "$(jq -r .artifacts.sparkcache_cuda_placement_sha256 "$receipt")" \ + --native-elf-manifest-sha256 "$(jq -r .runtime_contract.native_elf_manifest_sha256 "$receipt")" \ + --native-dispatch-manifest-sha256 "$(jq -r .runtime_contract.native_dispatch_manifest_sha256 "$receipt")" \ + --source-receipt-sha256 "$(jq -r .artifacts.source_receipt_sha256 "$receipt")" \ + --profile-output /path/to/glm53-public-python-overlay-profile.json \ + --site-output /path/to/glm53-public-python-overlay-site.yaml + +python scripts/sparkring_generic_launcher.py \ + --site /path/to/glm53-public-python-overlay-site.yaml \ + --profile /path/to/glm53-public-python-overlay-profile.json \ + plan +``` + +`plan` is offline and makes no SSH connection. Inspect all four rank actions +before any lifecycle command. The profile uses separate container, served +model, JIT cache, SparkCache root, MTP identity, and one-shot clear token names +so it cannot silently reuse the source-built runtime's test state. diff --git a/runtime/glm53-flash-adaptive-mtp-python-overlay/build-image.sh b/runtime/glm53-flash-adaptive-mtp-python-overlay/build-image.sh new file mode 100644 index 00000000..c78118b3 --- /dev/null +++ b/runtime/glm53-flash-adaptive-mtp-python-overlay/build-image.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Build a GLM-5.3 SparkCache image by replacing only attested Python sources. +set -euo pipefail + +here="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(git -C "${here}" rev-parse --show-toplevel)" +pins="${here}/pins.json" +engine="${CONTAINER_ENGINE:-docker}" +image="${IMAGE:-sparkring-glm53-sparkcache:vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64}" +receipt_path="${BUILD_RECEIPT:-${PWD}/glm53-public-python-overlay-image-receipt.json}" + +fatal() { + printf 'FATAL: %s\n' "$*" >&2 + exit 78 +} + +read_pin() { + python3 - "${pins}" "$1" <<'PY' +import json +import sys + +value = json.load(open(sys.argv[1], encoding="utf-8")) +for component in sys.argv[2].split("."): + value = value[int(component)] if isinstance(value, list) else value[component] +print(value) +PY +} + +tracked_inputs=( + runtime/glm53-flash-adaptive-mtp-python-overlay + LICENSE +) +git -C "${repo_root}" diff --quiet HEAD -- "${tracked_inputs[@]}" || + fatal "builder inputs differ from the checked-out SparkRing revision" +untracked="$(git -C "${repo_root}" ls-files --others --exclude-standard -- "${tracked_inputs[@]}")" +[[ -z "${untracked}" ]] || + fatal "builder inputs include untracked files: ${untracked%%$'\n'*}" + +public_base="$(read_pin public_base.reference)" +public_base_id="$(read_pin public_base.image_id)" +arm_builder="$(read_pin builder.arm_builder)" +vllm_native_commit="$(read_pin vllm.native_commit)" +vllm_python_commit="$(read_pin vllm.python_commit)" +vllm_python_tree="$(read_pin vllm.python_tree)" +overlay_manifest_sha256="$(read_pin vllm.overlay_manifest_sha256)" +b12x_commit="$(read_pin b12x.commit)" +b12x_tree="$(read_pin b12x.tree)" +sparkcache_commit="$(read_pin sparkcache.commit)" +sparkcache_tree="$(read_pin sparkcache.tree)" +sparkcache_source_sha256="$(read_pin sparkcache.source_tree_sha256)" +sparkring_revision="$(git -C "${repo_root}" rev-parse HEAD)" + +"${engine}" pull --platform linux/arm64 "${public_base}" +python3 "${here}/verify_image.py" \ + --engine "${engine}" --pins "${pins}" --base-image "${public_base}" >/dev/null + +workspace="$(mktemp -d)" +context="${workspace}/context" +cleanup() { + # `workspace` is created by mktemp in this process and never accepts caller input. + rm -rf -- "${workspace}" +} +trap cleanup EXIT + +python3 "${here}/prepare_context.py" \ + --repo-root "${repo_root}" "${context}" >/dev/null +python3 "${here}/prepare_context.py" --verify "${context}" >/dev/null +source_receipt_sha256="$(sha256sum "${context}/receipt.json" | cut -d' ' -f1)" +mkdir -p "${context}/base-probe" +"${engine}" run --rm --entrypoint python3 \ + --volume "${here}:/contract:ro" \ + --volume "${context}/base-probe:/out" \ + "${public_base}" \ + /contract/overlay_contract.py \ + --pins /contract/pins.json \ + --manifest /contract/vllm-python-overlay.json \ + record-base \ + --site-root /usr/local/lib/python3.12/dist-packages \ + --console-script /usr/local/bin/vllm \ + --output /out/retained-native.json >/dev/null +native_elf_manifest_sha256="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8"))["native_elf_manifest_sha256"])' "${context}/base-probe/retained-native.json")" +native_dispatch_manifest_sha256="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8"))["native_dispatch_manifest_sha256"])' "${context}/base-probe/retained-native.json")" +cuda_placement_stage="sparkring-sparkcache-cuda-placement:${sparkcache_commit:0:12}-${sparkring_revision:0:12}" +"${engine}" build \ + --platform linux/arm64 \ + --target sparkcache-cuda-placement \ + --file "${context}/bundle/runtime/Containerfile" \ + --build-arg "ARM_BUILDER=${arm_builder}" \ + --tag "${cuda_placement_stage}" \ + "${context}" +sparkcache_cuda_placement_sha256="$("${engine}" run --rm --entrypoint sha256sum \ + "${cuda_placement_stage}" \ + /build/sparkcache-cuda-placement/build-cuda/libspark_cache_placement.so | cut -d' ' -f1)" + +"${engine}" build \ + --platform linux/arm64 \ + --file "${context}/bundle/runtime/Containerfile" \ + --build-arg "PUBLIC_BASE=${public_base}" \ + --build-arg "PUBLIC_BASE_ID=${public_base_id}" \ + --build-arg "ARM_BUILDER=${arm_builder}" \ + --build-arg "VLLM_NATIVE_COMMIT=${vllm_native_commit}" \ + --build-arg "VLLM_PYTHON_COMMIT=${vllm_python_commit}" \ + --build-arg "VLLM_PYTHON_TREE=${vllm_python_tree}" \ + --build-arg "B12X_COMMIT=${b12x_commit}" \ + --build-arg "B12X_TREE=${b12x_tree}" \ + --build-arg "SPARKCACHE_COMMIT=${sparkcache_commit}" \ + --build-arg "SPARKCACHE_TREE=${sparkcache_tree}" \ + --build-arg "SPARKCACHE_SOURCE_SHA256=${sparkcache_source_sha256}" \ + --build-arg "SPARKRING_REVISION=${sparkring_revision}" \ + --build-arg "SOURCE_RECEIPT_SHA256=${source_receipt_sha256}" \ + --build-arg "OVERLAY_MANIFEST_SHA256=${overlay_manifest_sha256}" \ + --build-arg "NATIVE_ELF_MANIFEST_SHA256=${native_elf_manifest_sha256}" \ + --build-arg "NATIVE_DISPATCH_MANIFEST_SHA256=${native_dispatch_manifest_sha256}" \ + --build-arg "SPARKCACHE_CUDA_PLACEMENT_SHA256=${sparkcache_cuda_placement_sha256}" \ + --tag "${image}" \ + "${context}" + +python3 "${here}/verify_image.py" \ + --engine "${engine}" --pins "${pins}" --image "${image}" \ + --output "${receipt_path}" >/dev/null +printf 'image=%s\nreceipt=%s\n' "${image}" "${receipt_path}" +"${engine}" image inspect --format '{{.Id}}' "${image}" diff --git a/runtime/glm53-flash-adaptive-mtp-python-overlay/overlay_contract.py b/runtime/glm53-flash-adaptive-mtp-python-overlay/overlay_contract.py new file mode 100644 index 00000000..ba492fa0 --- /dev/null +++ b/runtime/glm53-flash-adaptive-mtp-python-overlay/overlay_contract.py @@ -0,0 +1,576 @@ +#!/usr/bin/env python3 +"""Verify the composed vLLM Python and retained-native runtime contract.""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import importlib.metadata +import inspect +import json +from dataclasses import fields +from pathlib import Path +from typing import Any, Iterable + + +PINS_SCHEMA = "sparkring-glm53-public-python-overlay/v1" +OVERLAY_SCHEMA = "sparkring-vllm-python-overlay/v1" +BASE_RECORD_SCHEMA = "sparkring-vllm-retained-native/v1" +SHA256_HEX_LENGTH = 64 +ELF_MAGIC = b"\x7fELF" +DISPATCH_PREFIXES = ("vllm::", "_C::", "_C_cache_ops::") + + +class ContractError(RuntimeError): + """The composed runtime differs from its byte-exact contract.""" + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_sha256(value: Any) -> str: + payload = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + return sha256_bytes(payload) + + +def load_json(path: Path, schema: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ContractError(f"cannot read {path}: {exc}") from exc + if not isinstance(value, dict) or value.get("schema") != schema: + raise ContractError(f"{path} does not use schema {schema}") + return value + + +def safe_relative_path(value: Any) -> Path: + if not isinstance(value, str) or not value or "\\" in value: + raise ContractError(f"invalid overlay path: {value!r}") + path = Path(value) + if path.is_absolute() or ".." in path.parts or path.parts[0] != "vllm": + raise ContractError(f"overlay path escapes the vLLM package: {value!r}") + return path + + +def _sha256(value: Any, label: str) -> str: + if ( + not isinstance(value, str) + or len(value) != SHA256_HEX_LENGTH + or any(character not in "0123456789abcdef" for character in value) + ): + raise ContractError(f"{label} must be a lowercase SHA-256") + return value + + +def validate_overlay_manifest(manifest: dict[str, Any]) -> tuple[dict[str, Any], ...]: + files = manifest.get("files") + if not isinstance(files, list) or len(files) != 31: + raise ContractError("the vLLM Python overlay must contain exactly 31 files") + records: list[dict[str, Any]] = [] + paths: set[str] = set() + additions = 0 + for index, raw in enumerate(files): + if not isinstance(raw, dict): + raise ContractError(f"overlay file record {index} is not an object") + relative = safe_relative_path(raw.get("path")) + normalized = relative.as_posix() + if normalized in paths: + raise ContractError(f"duplicate overlay path: {normalized}") + paths.add(normalized) + operation = raw.get("operation") + if operation not in ("add", "replace"): + raise ContractError(f"unsupported overlay operation for {normalized}") + base_hash = raw.get("base_sha256") + if operation == "add": + additions += 1 + if base_hash is not None: + raise ContractError(f"added overlay path has a base hash: {normalized}") + else: + _sha256(base_hash, f"{normalized} base hash") + _sha256(raw.get("target_sha256"), f"{normalized} target hash") + target_bytes = raw.get("target_bytes") + if not isinstance(target_bytes, int) or target_bytes <= 0: + raise ContractError(f"invalid target byte count for {normalized}") + records.append(raw) + if additions != 1: + raise ContractError("the vLLM Python overlay must add exactly one file") + return tuple(records) + + +def final_file_hashes(pins: dict[str, Any]) -> dict[str, str]: + result: dict[str, str] = {} + for patch in pins["sparkcache"]["patches"]: + target = safe_relative_path(patch["target"]).as_posix() + result[target] = _sha256( + patch["postimage_sha256"], f"{target} patch postimage" + ) + return result + + +def validate_dflash_loader_source(source: str) -> None: + """Require DFlash to pass its optional draft LoadConfig to get_model.""" + + tree = ast.parse(source) + function = next( + ( + node + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == "load_dflash_model" + ), + None, + ) + if function is None: + raise ContractError("DFlash loader source omits load_dflash_model") + calls = [ + node + for node in ast.walk(function) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "get_model" + ] + if len(calls) != 1: + raise ContractError("DFlash loader must contain exactly one get_model call") + keyword = next( + (item for item in calls[0].keywords if item.arg == "load_config"), + None, + ) + value = None if keyword is None else keyword.value + if not ( + isinstance(value, ast.Attribute) + and value.attr == "draft_load_config" + and isinstance(value.value, ast.Name) + and value.value.id == "speculative_config" + ): + raise ContractError( + "DFlash get_model must consume speculative_config.draft_load_config" + ) + + +def validate_optional_load_config_fallback(source: str) -> None: + """Require get_model(None) to retain the enclosing vLLM LoadConfig.""" + + tree = ast.parse(source) + function = next( + ( + node + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == "get_model" + ), + None, + ) + if function is None: + raise ContractError("model-loader source omits get_model") + calls = [ + node + for node in ast.walk(function) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "get_model_loader" + ] + if len(calls) != 1 or len(calls[0].args) != 1: + raise ContractError("get_model must select exactly one model loader") + selected = calls[0].args[0] + if not ( + isinstance(selected, ast.BoolOp) + and isinstance(selected.op, ast.Or) + and len(selected.values) == 2 + and isinstance(selected.values[0], ast.Name) + and selected.values[0].id == "load_config" + and isinstance(selected.values[1], ast.Attribute) + and selected.values[1].attr == "load_config" + and isinstance(selected.values[1].value, ast.Name) + and selected.values[1].value.id == "vllm_config" + ): + raise ContractError( + "get_model must fall back from a missing draft LoadConfig to " + "vllm_config.load_config" + ) + + +def verify_vllm_runtime_patch_files( + root: Path, + pins: dict[str, Any], +) -> list[dict[str, str]]: + """Verify installed runtime-patch postimages and loader semantics.""" + + verified = [] + for record in pins["vllm"].get("runtime_patches", ()): + relative = safe_relative_path(record["target"]) + path = root / relative + observed = sha256_file(path) + if observed != record["postimage_sha256"]: + raise ContractError( + f"vLLM runtime patch postimage mismatch for {relative}: " + f"expected {record['postimage_sha256']}, got {observed}" + ) + compile(path.read_bytes(), str(path), "exec") + if relative.as_posix() == "vllm/v1/worker/gpu/spec_decode/dflash/utils.py": + validate_dflash_loader_source(path.read_text(encoding="utf-8")) + verified.append( + { + "path": relative.as_posix(), + "sha256": observed, + } + ) + loader = root / "vllm/model_executor/model_loader/__init__.py" + validate_optional_load_config_fallback(loader.read_text(encoding="utf-8")) + return verified + + +def verify_overlay_files( + root: Path, + manifest: dict[str, Any], + *, + stage: str, + pins: dict[str, Any] | None = None, +) -> list[Path]: + records = validate_overlay_manifest(manifest) + if stage not in ("base", "target", "final"): + raise ContractError(f"unsupported overlay verification stage: {stage}") + final_hashes = final_file_hashes(pins) if stage == "final" and pins else {} + verified: list[Path] = [] + root = root.resolve() + for record in records: + relative = safe_relative_path(record["path"]) + path = (root / relative).resolve() + try: + path.relative_to(root) + except ValueError as exc: + raise ContractError(f"overlay path escapes root: {relative}") from exc + if stage == "base" and record["operation"] == "add": + if path.exists(): + raise ContractError(f"base image unexpectedly contains {relative}") + continue + if not path.is_file(): + raise ContractError(f"overlay file is absent: {relative}") + expected = ( + record["base_sha256"] + if stage == "base" + else final_hashes.get(relative.as_posix(), record["target_sha256"]) + ) + observed = sha256_file(path) + if observed != expected: + raise ContractError( + f"{stage} vLLM source mismatch for {relative}: " + f"expected {expected}, got {observed}" + ) + if stage != "base" and path.stat().st_size != record["target_bytes"]: + if relative.as_posix() not in final_hashes: + raise ContractError(f"target byte count mismatch for {relative}") + verified.append(path) + return verified + + +def elf_manifest(package_root: Path) -> list[dict[str, Any]]: + package_root = package_root.resolve() + records: list[dict[str, Any]] = [] + for path in sorted(item for item in package_root.rglob("*") if item.is_file()): + with path.open("rb") as handle: + magic = handle.read(len(ELF_MAGIC)) + if magic != ELF_MAGIC: + continue + records.append( + { + "path": path.relative_to(package_root).as_posix(), + "bytes": path.stat().st_size, + "sha256": sha256_file(path), + } + ) + if not records: + raise ContractError(f"no ELF files found below {package_root}") + return records + + +def distribution_metadata_manifest(site_root: Path, console_script: Path) -> dict[str, Any]: + candidates = sorted(site_root.glob("vllm-*.dist-info")) + if len(candidates) != 1: + raise ContractError( + f"expected one vLLM dist-info directory below {site_root}, got {len(candidates)}" + ) + dist_info = candidates[0] + files = [] + for path in sorted(item for item in dist_info.rglob("*") if item.is_file()): + files.append( + { + "path": path.relative_to(site_root).as_posix(), + "bytes": path.stat().st_size, + "sha256": sha256_file(path), + } + ) + version_file = site_root / "vllm" / "_version.py" + for path, label in ((version_file, "vLLM generated version"), (console_script, "vLLM console script")): + if not path.is_file(): + raise ContractError(f"{label} is absent: {path}") + return { + "distribution_version": importlib.metadata.version("vllm"), + "dist_info": dist_info.name, + "files": files, + "version_file_sha256": sha256_file(version_file), + "console_script": str(console_script), + "console_script_sha256": sha256_file(console_script), + } + + +def dispatch_manifest() -> list[str]: + try: + import torch + import vllm._custom_ops # noqa: F401 + except Exception as exc: + raise ContractError(f"cannot load retained vLLM native operators: {exc}") from exc + names = sorted( + name + for name in torch._C._dispatch_get_all_op_names() + if name.startswith(DISPATCH_PREFIXES) + ) + if not names: + raise ContractError("vLLM native dispatch operator set is empty") + return names + + +def make_base_record(site_root: Path, console_script: Path) -> dict[str, Any]: + native = elf_manifest(site_root / "vllm") + dispatch = dispatch_manifest() + metadata = distribution_metadata_manifest(site_root, console_script) + return { + "schema": BASE_RECORD_SCHEMA, + "native_elf": native, + "native_elf_manifest_sha256": canonical_sha256(native), + "native_dispatch": dispatch, + "native_dispatch_manifest_sha256": canonical_sha256(dispatch), + "distribution_metadata": metadata, + "distribution_metadata_manifest_sha256": canonical_sha256(metadata), + } + + +def verify_retained_native( + site_root: Path, console_script: Path, base_record: dict[str, Any] +) -> None: + if base_record.get("schema") != BASE_RECORD_SCHEMA: + raise ContractError("retained-native record uses an unsupported schema") + current = make_base_record(site_root, console_script) + for key in ( + "native_elf_manifest_sha256", + "native_dispatch_manifest_sha256", + "distribution_metadata_manifest_sha256", + ): + if current[key] != base_record.get(key): + raise ContractError( + f"retained vLLM {key} mismatch: " + f"expected {base_record.get(key)}, got {current[key]}" + ) + + +def verify_b12x_contract(pins: dict[str, Any]) -> dict[str, Any]: + try: + import b12x + from b12x.gemm import bf16_vocab_projection + from b12x.sequence import gdn_decode + except Exception as exc: + raise ContractError(f"cannot import the pinned B12X interfaces: {exc}") from exc + version = importlib.metadata.version("b12x") + expected_version = pins["b12x"]["package_version"] + if version != expected_version: + raise ContractError(f"B12X version mismatch: expected {expected_version}, got {version}") + field_names, bind_parameters = validate_b12x_surface( + gdn_decode.Caps, + gdn_decode.bind_kda, + bf16_vocab_projection, + required_field=pins["b12x"]["required_caps_field"], + ) + caps = gdn_decode.Caps( + device="cuda:0", + max_tokens=1, + max_seqs=1, + max_state_slots=1, + key_heads=1, + value_heads=1, + gate_activation="sigmoid", + kda_metadata_validation=pins["b12x"]["required_caps_value"], + ) + if caps.kda_metadata_validation != pins["b12x"]["required_caps_value"]: + raise ContractError("B12X Caps did not retain trusted metadata validation") + return { + "version": version, + "module": str(Path(b12x.__file__).resolve()), + "caps_fields": sorted(field_names), + "bind_kda_parameters": list(bind_parameters), + } + + +def validate_b12x_surface( + caps_type: Any, + bind_kda: Any, + bf16_vocab_projection: Any, + *, + required_field: str, +) -> tuple[set[str], tuple[str, ...]]: + field_names = {item.name for item in fields(caps_type)} + if required_field not in field_names: + raise ContractError(f"B12X Caps omits required field {required_field}") + bind_parameters = tuple(inspect.signature(bind_kda).parameters) + required_bind_parameters = { + "mixed_qkv", + "raw_g", + "raw_beta", + "z", + "query_start_loc", + "num_accepted_tokens", + "state_indices", + "num_seqs", + "num_tokens", + "output", + } + missing = sorted(required_bind_parameters.difference(bind_parameters)) + if missing: + raise ContractError(f"B12X bind_kda omits live tensor parameters: {missing}") + if not hasattr(bf16_vocab_projection, "plan"): + raise ContractError("B12X BF16 vocabulary projection omits plan") + return field_names, bind_parameters + + +def verify_dependencies(pins: dict[str, Any]) -> dict[str, str]: + expected = pins["dependencies"] + observed = { + "torch": importlib.metadata.version("torch"), + "fastsafetensors": importlib.metadata.version("fastsafetensors"), + "instanttensor": importlib.metadata.version("instanttensor"), + } + for name, version in observed.items(): + if version != expected[name]: + raise ContractError( + f"{name} version mismatch: expected {expected[name]}, got {version}" + ) + nccl = Path(expected["nccl_library"]) + if not nccl.is_file(): + raise ContractError(f"pinned NCCL library is absent: {nccl}") + observed_nccl = sha256_file(nccl) + if observed_nccl != expected["nccl_library_sha256"]: + raise ContractError( + f"NCCL library mismatch: expected {expected['nccl_library_sha256']}, " + f"got {observed_nccl}" + ) + observed["nccl_sha256"] = observed_nccl + return observed + + +def compile_overlay_files(root: Path, manifest: dict[str, Any]) -> None: + for record in validate_overlay_manifest(manifest): + path = root / safe_relative_path(record["path"]) + compile(path.read_bytes(), str(path), "exec") + + +def composed_report( + *, + root: Path, + site_root: Path, + console_script: Path, + pins: dict[str, Any], + manifest: dict[str, Any], + base_record: dict[str, Any], +) -> dict[str, Any]: + verified = verify_overlay_files(root, manifest, stage="final", pins=pins) + compile_overlay_files(root, manifest) + verify_retained_native(site_root, console_script, base_record) + runtime_patches = verify_vllm_runtime_patch_files(root, pins) + return { + "schema": "sparkring-glm53-public-python-overlay-verification/v1", + "status": "implemented", + "vllm_python_files_verified": len(verified), + "vllm_python_commit": pins["vllm"]["python_commit"], + "vllm_native_commit": pins["vllm"]["native_commit"], + "vllm_runtime_patches": runtime_patches, + "b12x": verify_b12x_contract(pins), + "dependencies": verify_dependencies(pins), + "native_elf_manifest_sha256": base_record["native_elf_manifest_sha256"], + "native_dispatch_manifest_sha256": base_record[ + "native_dispatch_manifest_sha256" + ], + "distribution_metadata_manifest_sha256": base_record[ + "distribution_metadata_manifest_sha256" + ], + } + + +def _write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + + +def main(argv: Iterable[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pins", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + subparsers = parser.add_subparsers(dest="command", required=True) + + verify_files = subparsers.add_parser("verify-files") + verify_files.add_argument("--root", type=Path, required=True) + verify_files.add_argument("--stage", choices=("base", "target", "final"), required=True) + + record_base = subparsers.add_parser("record-base") + record_base.add_argument("--site-root", type=Path, required=True) + record_base.add_argument("--console-script", type=Path, default=Path("/usr/local/bin/vllm")) + record_base.add_argument("--output", type=Path, required=True) + + verify_composed = subparsers.add_parser("verify-composed") + verify_composed.add_argument("--root", type=Path, required=True) + verify_composed.add_argument("--site-root", type=Path, required=True) + verify_composed.add_argument("--console-script", type=Path, default=Path("/usr/local/bin/vllm")) + verify_composed.add_argument("--base-record", type=Path, required=True) + verify_composed.add_argument("--output", type=Path) + + args = parser.parse_args(list(argv) if argv is not None else None) + try: + pins = load_json(args.pins, PINS_SCHEMA) + manifest = load_json(args.manifest, OVERLAY_SCHEMA) + if args.command == "verify-files": + verified = verify_overlay_files( + args.root, + manifest, + stage=args.stage, + pins=pins, + ) + report: dict[str, Any] = { + "stage": args.stage, + "verified": len(verified), + } + elif args.command == "record-base": + report = make_base_record(args.site_root, args.console_script) + _write_json(args.output, report) + else: + base_record = load_json(args.base_record, BASE_RECORD_SCHEMA) + report = composed_report( + root=args.root, + site_root=args.site_root, + console_script=args.console_script, + pins=pins, + manifest=manifest, + base_record=base_record, + ) + if args.output: + _write_json(args.output, report) + except (ContractError, KeyError, OSError) as exc: + parser.error(str(exc)) + print(json.dumps(report, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/runtime/glm53-flash-adaptive-mtp-python-overlay/patches/010-dflash-draft-load-config.patch b/runtime/glm53-flash-adaptive-mtp-python-overlay/patches/010-dflash-draft-load-config.patch new file mode 100644 index 00000000..4681359a --- /dev/null +++ b/runtime/glm53-flash-adaptive-mtp-python-overlay/patches/010-dflash-draft-load-config.patch @@ -0,0 +1,13 @@ +diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py +index b9e6ea02f..84c7d54ac 100644 +--- a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py ++++ b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py +@@ -46,5 +46,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.M + ) + with set_model_tag("dflash_head"): + dflash_model = get_model( +- vllm_config=draft_vllm_config, model_config=draft_model_config ++ vllm_config=draft_vllm_config, ++ model_config=draft_model_config, ++ load_config=speculative_config.draft_load_config, + ) diff --git a/runtime/glm53-flash-adaptive-mtp-python-overlay/pins.json b/runtime/glm53-flash-adaptive-mtp-python-overlay/pins.json new file mode 100644 index 00000000..1d69d933 --- /dev/null +++ b/runtime/glm53-flash-adaptive-mtp-python-overlay/pins.json @@ -0,0 +1,132 @@ +{ + "schema": "sparkring-glm53-public-python-overlay/v1", + "status": "implemented", + "qualification": "The builder and offline verifier are implemented. The resulting image is unsupported for serving until an immutable image digest passes four-rank TP4/DCP1 model loading, generation, SparkCache restore, failure-recovery, and concurrency checks.", + "platform": "linux/arm64", + "public_base": { + "reference": "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd", + "image_id": "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5", + "labels": { + "org.jovian.architecture": "linux-arm64-sm121", + "org.jovian.vllm.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.jovian.b12x.commit": "2fcf23a0ce269be27b2e03fece73d46e90e6aeea", + "org.jovian.transport": "sparkring-nccl-2.30.7-source-built", + "org.sparkring.nccl.commit": "73cf112295c33aee2b895f329f592f2a9b4b0f97", + "org.sparkring.nccl.patched-tree": "abdeb053b94c3f6d472cd55ae2b79ca821299009", + "org.sparkring.nccl.patch-sha256": "6709063fa1c25055ae77a9397dea5d89643f8211d25e7990bdd11597d08c0dde" + } + }, + "builder": { + "arm_builder": "pytorch/manylinuxaarch64-builder@sha256:f91599c49f526c77d01b68286f2bf943a5fd6a432d7e3f0afcc5784825908fe9", + "output_name": "sparkring-glm53-sparkcache:vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64" + }, + "vllm": { + "repository": "https://github.com/local-inference-lab/vllm.git", + "native_commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "native_tree": "4a99033352bc1a6b00852dc0e8b1cecbdd9f0ebb", + "python_commit": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "python_tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "overlay_manifest": "runtime/glm53-flash-adaptive-mtp-python-overlay/vllm-python-overlay.json", + "overlay_manifest_sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a", + "runtime_patches": [ + { + "status": "implemented", + "path": "runtime/glm53-flash-adaptive-mtp-python-overlay/patches/010-dflash-draft-load-config.patch", + "target": "vllm/v1/worker/gpu/spec_decode/dflash/utils.py", + "sha256": "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279", + "preimage_sha256": "2301c8199b73ed893dfbd3ae14ad125816f100b2d2ed034215b1f2d9aa2c23c5", + "postimage_sha256": "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4", + "contract": "DFlash passes SpeculativeConfig.draft_load_config to get_model; None retains the target LoadConfig fallback." + } + ], + "native_source_objects": { + "csrc": "9ada29088768f1bc08dadd2eed3c9738eb9ac8a1", + "cmake": "5e5bbdbe1c1b3a479656d8d6a41cc32a1982c43d", + "rust": "85c3cd52db223217d45377d3f7f884e756641de3", + "CMakeLists.txt": "bb0f51b43ef4e1c57918b551b8cf213f9059b601", + "setup.py": "ae64a13daa0f1facc255afcdb4ffcad264776b98", + "pyproject.toml": "0766645fc7481da0ec439208128b838b3348d94c", + "requirements": "d6e1c8e13cd4c4358ab422e3ef006d3f9f23e18b", + "docker/Dockerfile": "6e20f6eab482782ed90a05d91b669b59641eaa46" + }, + "retained_distribution_metadata": [ + "vllm/_version.py", + "vllm-*.dist-info", + "/usr/local/bin/vllm" + ] + }, + "b12x": { + "repository": "https://github.com/local-inference-lab/b12x.git", + "commit": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "package_version": "1.3.0", + "base_commit": "2fcf23a0ce269be27b2e03fece73d46e90e6aeea", + "base_tree": "58a046fc8faa747346f40f87166cda7e0f67ff47", + "required_caps_field": "kda_metadata_validation", + "required_caps_value": "trusted" + }, + "dependencies": { + "python": "3.12", + "torch": "2.13.0+cu130", + "cuda": "13.0.3", + "fastsafetensors": "0.3.3", + "instanttensor": "0.1.9", + "instanttensor_sdist_sha256": "d8692b97991c1a5fb2db7905b9a6ae90a7f967c7ddd853d35e41caa146750c02", + "nccl_library": "/opt/sparkring/nccl/libnccl.so.2.30.7", + "nccl_library_sha256": "5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3" + }, + "sparkcache": { + "repository": "https://github.com/FujitsuPolycom/sparkcache.git", + "commit": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "tree": "e864ed9ad64f771188fdb59aa9738e348134d636", + "source_tree_sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", + "contract": { + "path": "sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json", + "sha256": "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024", + "files": 11 + }, + "patches": [ + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/020-sparkcache-vmm-exemption.patch", + "sha256": "370b498eebf44b4e52a2d2751fa249ad4bd3d0b6fd951b063a161fb06febbe99", + "target": "vllm/config/vllm.py", + "preimage_sha256": "cc03756d9bebf2a128828e5fbf7e9766446884dfce016c34381bce0aa78bfd9e", + "postimage_sha256": "9f64f5041f7f9d953e9f6bc53de8733b3eb4035c0753056a1f646346702a0994" + }, + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/030-sparkcache-hma-load-failure.patch", + "sha256": "0202df4b5db7bd35540eebdef51fcbe2bb01845b952964080e1fe903ee19b404", + "target": "vllm/v1/core/sched/scheduler.py", + "preimage_sha256": "05c05f4b372c7a4bf76399b38b338eab657c69406b7019ab02101d2ab0c7764c", + "postimage_sha256": "337893b6b088d12eb38d8d70c866242d085134442289b4f4574a1f162c9f11c8" + }, + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/040-sparkcache-shared-prefix-lease.patch", + "sha256": "6c6d6bdc2d6e35742ef37715e88697b1f972d23b28948690c4d8d835edcaf01b", + "target": "vllm/v1/core/kv_cache_manager.py", + "preimage_sha256": "02c71da26bbac81629248ee42b0a71bd2db817d339894d647d2ff6b66fd5ad19", + "postimage_sha256": "ee03dc9ce2b720c0be6e9f572d23580ba96eff68fe3406250557e83071654af0" + }, + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/041-sparkcache-shared-prefix-attach.patch", + "sha256": "b98e6bc06990f608fc5f0828c11b8eb453fbec0f8fbbf24ba45254810b7e67c3", + "target": "vllm/v1/core/sched/scheduler.py", + "preimage_sha256": "337893b6b088d12eb38d8d70c866242d085134442289b4f4574a1f162c9f11c8", + "postimage_sha256": "6d397c97f31e67a75efc01b5ddd89fa58db425de14fa43965ef2d6146b6b9bdb" + } + ] + }, + "runtime_cache_roots": { + "VLLM_CACHE_ROOT": "/cache/jit/vllm/py-0b67266-native-da4d7be", + "B12X_CUTE_COMPILE_CACHE_DIR": "/cache/jit/b12x/b1d541f9/cute", + "TRITON_CACHE_DIR": "/cache/jit/triton/vllm-py-0b67266-native-da4d7be" + }, + "outputs": { + "image": null, + "image_id": null, + "image_digest": null, + "native_elf_manifest_sha256": null, + "native_dispatch_manifest_sha256": null, + "b12x_wheel_sha256": null + } +} diff --git a/runtime/glm53-flash-adaptive-mtp-python-overlay/prepare_context.py b/runtime/glm53-flash-adaptive-mtp-python-overlay/prepare_context.py new file mode 100644 index 00000000..0167b0dc --- /dev/null +++ b/runtime/glm53-flash-adaptive-mtp-python-overlay/prepare_context.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +"""Prepare exact public sources for the GLM-5.3 Python-overlay image.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import shutil +import subprocess +from pathlib import Path +from typing import Any, Iterable + + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +PINS = HERE / "pins.json" +MANIFEST = HERE / "vllm-python-overlay.json" +PINS_SCHEMA = "sparkring-glm53-public-python-overlay/v1" +OVERLAY_SCHEMA = "sparkring-vllm-python-overlay/v1" +RECEIPT_SCHEMA = "sparkring-glm53-public-python-overlay-context/v1" + + +class PrepareError(RuntimeError): + """A source input differs from the public Python-overlay contract.""" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run(argv: Iterable[str], *, cwd: Path | None = None) -> str: + arguments = list(argv) + environment = os.environ.copy() + environment["GIT_TERMINAL_PROMPT"] = "0" + completed = subprocess.run( + arguments, + cwd=cwd, + env=environment, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if completed.returncode: + detail = completed.stderr.strip() or completed.stdout.strip() + raise PrepareError(f"command failed ({' '.join(arguments)}): {detail}") + return completed.stdout.strip() + + +def load_json(path: Path, schema: str) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict) or value.get("schema") != schema: + raise PrepareError(f"{path} does not use schema {schema}") + return value + + +def clone_detached( + destination: Path, + *, + repository: str, + commit: str, + tree: str, + fetch_depth: int, +) -> None: + destination.mkdir(parents=True) + run(("git", "init", "--quiet", str(destination))) + run(("git", "-C", str(destination), "config", "core.autocrlf", "false")) + run(("git", "-C", str(destination), "config", "core.longpaths", "true")) + run(("git", "-C", str(destination), "remote", "add", "origin", repository)) + run( + ( + "git", + "-C", + str(destination), + "fetch", + "--quiet", + "--depth", + str(fetch_depth), + "origin", + commit, + ) + ) + run(("git", "-C", str(destination), "checkout", "--quiet", "--detach", "FETCH_HEAD")) + verify_git_source(destination, commit=commit, tree=tree) + + +def verify_git_source(source: Path, *, commit: str, tree: str) -> None: + observed_commit = run(("git", "-C", str(source), "rev-parse", "HEAD")) + observed_tree = run(("git", "-C", str(source), "rev-parse", "HEAD^{tree}")) + if observed_commit != commit: + raise PrepareError( + f"{source.name} commit mismatch: expected {commit}, got {observed_commit}" + ) + if observed_tree != tree: + raise PrepareError( + f"{source.name} tree mismatch: expected {tree}, got {observed_tree}" + ) + if run(("git", "-C", str(source), "status", "--porcelain")): + raise PrepareError(f"{source.name} source tree is not clean") + + +def _load_overlay_contract(): + path = HERE / "overlay_contract.py" + spec = importlib.util.spec_from_file_location("glm53_overlay_contract", path) + if spec is None or spec.loader is None: + raise PrepareError(f"cannot load overlay contract from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def verify_vllm_lineage( + source: Path, pins: dict[str, Any], manifest: dict[str, Any] +) -> None: + overlay = _load_overlay_contract() + records = overlay.validate_overlay_manifest(manifest) + vllm = pins["vllm"] + base = vllm["native_commit"] + target = vllm["python_commit"] + if run(("git", "-C", str(source), "merge-base", base, target)) != base: + raise PrepareError("the retained-native vLLM commit is not the overlay base") + if run(("git", "-C", str(source), "rev-parse", f"{base}^{{tree}}")) != vllm[ + "native_tree" + ]: + raise PrepareError("the retained-native vLLM Git tree differs from its pin") + + observed: dict[str, str] = {} + diff = run(("git", "-C", str(source), "diff", "--name-status", f"{base}..{target}", "--", "vllm")) + for line in diff.splitlines(): + status, path = line.split("\t", 1) + observed[path] = "add" if status == "A" else "replace" if status == "M" else status + expected = {record["path"]: record["operation"] for record in records} + if observed != expected: + raise PrepareError("the vLLM Python delta differs from the 31-file allowlist") + + for path, expected_object in vllm["native_source_objects"].items(): + base_object = run(("git", "-C", str(source), "rev-parse", f"{base}:{path}")) + target_object = run(("git", "-C", str(source), "rev-parse", f"{target}:{path}")) + if base_object != expected_object or target_object != expected_object: + raise PrepareError(f"vLLM native build input changed: {path}") + + for record in records: + path = record["path"] + target_bytes = subprocess.run( + ("git", "-C", str(source), "show", f"{target}:{path}"), + check=True, + stdout=subprocess.PIPE, + ).stdout + if hashlib.sha256(target_bytes).hexdigest() != record["target_sha256"]: + raise PrepareError(f"vLLM target blob mismatch: {path}") + if len(target_bytes) != record["target_bytes"]: + raise PrepareError(f"vLLM target byte count mismatch: {path}") + base_result = subprocess.run( + ("git", "-C", str(source), "show", f"{base}:{path}"), + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + if record["operation"] == "add": + if base_result.returncode == 0: + raise PrepareError(f"vLLM added path already exists in base: {path}") + elif ( + base_result.returncode != 0 + or hashlib.sha256(base_result.stdout).hexdigest() != record["base_sha256"] + ): + raise PrepareError(f"vLLM base blob mismatch: {path}") + + +def verify_vllm_runtime_patches( + source: Path, + pins: dict[str, Any], + patch_root: Path, +) -> None: + """Verify each exact-input runtime patch and restore the clean source.""" + + for record in pins["vllm"].get("runtime_patches", ()): + patch = patch_root / Path(record["path"]).name + target = source / record["target"] + if sha256_file(patch) != record["sha256"]: + raise PrepareError(f"vLLM runtime patch mismatch: {record['path']}") + if sha256_file(target) != record["preimage_sha256"]: + raise PrepareError( + f"vLLM runtime patch preimage mismatch: {record['target']}" + ) + run(("git", "-C", str(source), "apply", "--check", str(patch))) + run(("git", "-C", str(source), "apply", str(patch))) + try: + observed = sha256_file(target) + if observed != record["postimage_sha256"]: + raise PrepareError( + f"vLLM runtime patch postimage mismatch: {record['target']}" + ) + finally: + run(("git", "-C", str(source), "apply", "--reverse", str(patch))) + verify_git_source( + source, + commit=pins["vllm"]["python_commit"], + tree=pins["vllm"]["python_tree"], + ) + + +def copy_overlay(source: Path, destination: Path, manifest: dict[str, Any]) -> None: + target = manifest["target"]["commit"] + for record in manifest["files"]: + relative = Path(record["path"]) + output = destination / relative + output.parent.mkdir(parents=True, exist_ok=True) + payload = subprocess.run( + ("git", "-C", str(source), "show", f"{target}:{relative.as_posix()}"), + check=True, + stdout=subprocess.PIPE, + ).stdout + output.write_bytes(payload) + + +def sparkcache_source_sha256(root: Path) -> str: + digest = hashlib.sha256(b"sparkcache-source-tree/v1\x00") + count = 0 + for path in sorted( + (candidate for candidate in root.rglob("*") if candidate.is_file()), + key=lambda candidate: candidate.relative_to(root).as_posix(), + ): + relative = path.relative_to(root) + if ( + any(part in {"__pycache__", ".pytest_cache", "build"} for part in relative.parts) + or path.suffix in {".pyc", ".pyo"} + ): + continue + if path.is_symlink(): + raise PrepareError(f"SparkCache source contains a symlink: {relative}") + encoded = relative.as_posix().encode("utf-8") + digest.update(len(encoded).to_bytes(4, "little")) + digest.update(encoded) + normalized = path.read_bytes().replace(b"\r\n", b"\n") + digest.update(hashlib.sha256(normalized).digest()) + count += 1 + if not count: + raise PrepareError("SparkCache source tree is empty") + return digest.hexdigest() + + +def copy_file(source: Path, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + +def prepare(output: Path, *, repository_root: Path = ROOT) -> dict[str, Any]: + if output.exists(): + raise PrepareError(f"output already exists: {output}") + pins = load_json(PINS, PINS_SCHEMA) + manifest = load_json(MANIFEST, OVERLAY_SCHEMA) + if sha256_file(MANIFEST) != pins["vllm"]["overlay_manifest_sha256"]: + raise PrepareError("vLLM overlay manifest SHA-256 differs from pins.json") + + sources = output / "bundle" / "sources" + runtime = output / "bundle" / "runtime" + overlay = output / "bundle" / "vllm-overlay" + sources.mkdir(parents=True) + runtime.mkdir(parents=True) + + vllm = sources / "vllm" + clone_detached( + vllm, + repository=pins["vllm"]["repository"], + commit=pins["vllm"]["python_commit"], + tree=pins["vllm"]["python_tree"], + fetch_depth=13, + ) + verify_vllm_lineage(vllm, pins, manifest) + copy_overlay(vllm, overlay, manifest) + patch_root = HERE / "patches" + verify_vllm_runtime_patches(vllm, pins, patch_root) + + b12x = sources / "b12x" + clone_detached( + b12x, + repository=pins["b12x"]["repository"], + commit=pins["b12x"]["commit"], + tree=pins["b12x"]["tree"], + fetch_depth=1, + ) + + sparkcache = sources / "sparkcache" + clone_detached( + sparkcache, + repository=pins["sparkcache"]["repository"], + commit=pins["sparkcache"]["commit"], + tree=pins["sparkcache"]["tree"], + fetch_depth=1, + ) + observed_sparkcache = sparkcache_source_sha256(sparkcache / "sparkcache") + if observed_sparkcache != pins["sparkcache"]["source_tree_sha256"]: + raise PrepareError( + "SparkCache deployable source mismatch: expected " + f"{pins['sparkcache']['source_tree_sha256']}, got {observed_sparkcache}" + ) + for patch in pins["sparkcache"]["patches"]: + path = sparkcache / patch["path"] + if sha256_file(path) != patch["sha256"]: + raise PrepareError(f"SparkCache patch mismatch: {patch['path']}") + contract = sparkcache / pins["sparkcache"]["contract"]["path"] + if sha256_file(contract) != pins["sparkcache"]["contract"]["sha256"]: + raise PrepareError("SparkCache vLLM contract differs from its pin") + + for filename in ( + "pins.json", + "vllm-python-overlay.json", + "overlay_contract.py", + "verify_image.py", + "Containerfile", + "build-image.sh", + "README.md", + ): + copy_file(HERE / filename, runtime / filename) + runtime_patches = [] + for patch in pins["vllm"].get("runtime_patches", ()): + name = Path(patch["path"]).name + copy_file(patch_root / name, runtime / "patches" / name) + runtime_patches.append(f"bundle/runtime/patches/{name}") + copy_file(repository_root / "LICENSE", runtime / "SparkRing-LICENSE") + + receipt_inputs = tuple( + f"bundle/runtime/{name}" + for name in ( + "pins.json", + "vllm-python-overlay.json", + "overlay_contract.py", + "verify_image.py", + "Containerfile", + "build-image.sh", + "README.md", + "SparkRing-LICENSE", + ) + ) + tuple(runtime_patches) + receipt = { + "schema": RECEIPT_SCHEMA, + "status": "implemented", + "sources": { + "vllm_python": { + "commit": pins["vllm"]["python_commit"], + "tree": pins["vllm"]["python_tree"], + "files": len(manifest["files"]), + }, + "vllm_native": { + "commit": pins["vllm"]["native_commit"], + "tree": pins["vllm"]["native_tree"], + }, + "b12x": {"commit": pins["b12x"]["commit"], "tree": pins["b12x"]["tree"]}, + "sparkcache": { + "commit": pins["sparkcache"]["commit"], + "tree": pins["sparkcache"]["tree"], + "source_tree_sha256": observed_sparkcache, + }, + }, + "vllm_runtime_patches": pins["vllm"].get("runtime_patches", []), + "files": {relative: sha256_file(output / relative) for relative in receipt_inputs}, + } + (output / "receipt.json").write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + verify_context(output) + return receipt + + +def verify_context(context: Path) -> dict[str, Any]: + pins = load_json(context / "bundle/runtime/pins.json", PINS_SCHEMA) + manifest = load_json( + context / "bundle/runtime/vllm-python-overlay.json", OVERLAY_SCHEMA + ) + receipt = load_json(context / "receipt.json", RECEIPT_SCHEMA) + for relative, expected in receipt["files"].items(): + if sha256_file(context / relative) != expected: + raise PrepareError(f"prepared context file mismatch: {relative}") + verify_git_source( + context / "bundle/sources/vllm", + commit=pins["vllm"]["python_commit"], + tree=pins["vllm"]["python_tree"], + ) + verify_vllm_lineage(context / "bundle/sources/vllm", pins, manifest) + if receipt.get("vllm_runtime_patches") != pins["vllm"].get( + "runtime_patches", [] + ): + raise PrepareError("prepared context vLLM runtime patch receipt differs") + verify_vllm_runtime_patches( + context / "bundle/sources/vllm", + pins, + context / "bundle/runtime/patches", + ) + verify_git_source( + context / "bundle/sources/b12x", + commit=pins["b12x"]["commit"], + tree=pins["b12x"]["tree"], + ) + verify_git_source( + context / "bundle/sources/sparkcache", + commit=pins["sparkcache"]["commit"], + tree=pins["sparkcache"]["tree"], + ) + overlay = _load_overlay_contract() + overlay.verify_overlay_files( + context / "bundle/vllm-overlay", manifest, stage="target" + ) + return receipt + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--repo-root", type=Path, default=ROOT) + parser.add_argument("--verify", action="store_true") + args = parser.parse_args() + try: + result = ( + verify_context(args.output.resolve()) + if args.verify + else prepare(args.output.resolve(), repository_root=args.repo_root.resolve()) + ) + except (OSError, KeyError, json.JSONDecodeError, PrepareError) as exc: + parser.error(str(exc)) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/runtime/glm53-flash-adaptive-mtp-python-overlay/test_public_python_overlay.py b/runtime/glm53-flash-adaptive-mtp-python-overlay/test_public_python_overlay.py new file mode 100644 index 00000000..e4c0d777 --- /dev/null +++ b/runtime/glm53-flash-adaptive-mtp-python-overlay/test_public_python_overlay.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import re +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +HERE = Path(__file__).resolve().parent +PINS = HERE / "pins.json" +MANIFEST = HERE / "vllm-python-overlay.json" +DFLASH_PATCH = HERE / "patches/010-dflash-draft-load-config.patch" + + +def _module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +contract = _module("glm53_public_python_overlay_contract", HERE / "overlay_contract.py") +verify = _module("glm53_public_python_overlay_verify", HERE / "verify_image.py") +prepare = _module("glm53_public_python_overlay_prepare", HERE / "prepare_context.py") + + +def test_overlay_pins_public_base_and_mixed_vllm_provenance() -> None: + pins = json.loads(PINS.read_text(encoding="utf-8")) + assert pins["status"] == "implemented" + assert pins["public_base"]["reference"] == ( + "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:" + "864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd" + ) + assert pins["vllm"]["native_commit"] == ( + "da4d7be6c97434f6942292ed8abbf4b32dc44355" + ) + assert pins["vllm"]["python_commit"] == ( + "0b67266a0f37d6146a8403fb8482403c62f412d5" + ) + assert pins["b12x"]["commit"] == ( + "b1d541f9e71a35f030d45fae437630fff7507c2a" + ) + assert pins["b12x"]["tree"] == "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae" + assert pins["b12x"]["package_version"] == "1.3.0" + assert pins["b12x"]["commit"] != pins["b12x"]["base_commit"] + sparkcache = pins["sparkcache"] + assert sparkcache["commit"] == ( + "5d571018de5b63a9a90e5c11e6d6e86bbff4a957" + ) + assert sparkcache["tree"] == "e864ed9ad64f771188fdb59aa9738e348134d636" + assert sparkcache["source_tree_sha256"] == ( + "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88" + ) + assert pins["dependencies"]["torch"] == "2.13.0+cu130" + + +def test_overlay_manifest_is_the_exact_31_file_python_delta() -> None: + pins = json.loads(PINS.read_text(encoding="utf-8")) + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + records = contract.validate_overlay_manifest(manifest) + assert len(records) == 31 + assert sum(record["target_bytes"] for record in records) == 1_567_308 + assert [record["operation"] for record in records].count("add") == 1 + assert next(record for record in records if record["operation"] == "add")[ + "path" + ] == "vllm/v1/spec_decode/dynamic/acceptance_length.py" + assert hashlib.sha256(MANIFEST.read_bytes()).hexdigest() == pins["vllm"][ + "overlay_manifest_sha256" + ] + + +def test_dflash_loader_patch_binds_exact_0b_preimage_and_postimage() -> None: + pins = json.loads(PINS.read_text(encoding="utf-8")) + assert pins["vllm"]["runtime_patches"] == [ + { + "status": "implemented", + "path": ( + "runtime/glm53-flash-adaptive-mtp-python-overlay/patches/" + "010-dflash-draft-load-config.patch" + ), + "target": "vllm/v1/worker/gpu/spec_decode/dflash/utils.py", + "sha256": "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279", + "preimage_sha256": ( + "2301c8199b73ed893dfbd3ae14ad125816f100b2d2ed034215b1f2d9aa2c23c5" + ), + "postimage_sha256": ( + "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4" + ), + "contract": ( + "DFlash passes SpeculativeConfig.draft_load_config to get_model; " + "None retains the target LoadConfig fallback." + ), + } + ] + assert prepare.sha256_file(DFLASH_PATCH) == pins["vllm"]["runtime_patches"][ + 0 + ]["sha256"] + + +def test_dflash_loader_contract_honors_explicit_draft_load_config() -> None: + contract.validate_dflash_loader_source( + """ +def load_dflash_model(target_model, vllm_config): + speculative_config = vllm_config.speculative_config + return get_model( + vllm_config=vllm_config, + model_config=speculative_config.draft_model_config, + load_config=speculative_config.draft_load_config, + ) +""" + ) + + +def test_dflash_loader_contract_rejects_an_ignored_draft_load_config() -> None: + with pytest.raises(contract.ContractError, match="draft_load_config"): + contract.validate_dflash_loader_source( + """ +def load_dflash_model(target_model, vllm_config): + speculative_config = vllm_config.speculative_config + return get_model( + vllm_config=vllm_config, + model_config=speculative_config.draft_model_config, + ) +""" + ) + + +def test_missing_draft_load_config_uses_the_target_load_config_fallback() -> None: + contract.validate_optional_load_config_fallback( + """ +def get_model(*, vllm_config, load_config=None): + loader = get_model_loader(load_config or vllm_config.load_config) + return loader.load_model() +""" + ) + + with pytest.raises(contract.ContractError, match="fall back"): + contract.validate_optional_load_config_fallback( + """ +def get_model(*, vllm_config, load_config=None): + loader = get_model_loader(vllm_config.load_config) + return loader.load_model() +""" + ) + + +def test_documented_dflash_config_separates_target_and_draft_loaders() -> None: + guide = (HERE / "README.md").read_text(encoding="utf-8") + match = re.search(r"--speculative-config '([^']+)'", guide) + assert match is not None + speculative = json.loads(match.group(1)) + assert "--load-format fastsafetensors" in guide + assert speculative["method"] == "dflash" + assert speculative["model"] == "/mtp-draft" + assert speculative["num_speculative_tokens"] == 7 + assert speculative["draft_load_config"] == {"load_format": "safetensors"} + + +def test_native_build_inputs_are_identical_git_objects() -> None: + pins = json.loads(PINS.read_text(encoding="utf-8")) + assert pins["vllm"]["native_source_objects"] == { + "csrc": "9ada29088768f1bc08dadd2eed3c9738eb9ac8a1", + "cmake": "5e5bbdbe1c1b3a479656d8d6a41cc32a1982c43d", + "rust": "85c3cd52db223217d45377d3f7f884e756641de3", + "CMakeLists.txt": "bb0f51b43ef4e1c57918b551b8cf213f9059b601", + "setup.py": "ae64a13daa0f1facc255afcdb4ffcad264776b98", + "pyproject.toml": "0766645fc7481da0ec439208128b838b3348d94c", + "requirements": "d6e1c8e13cd4c4358ab422e3ef006d3f9f23e18b", + "docker/Dockerfile": "6e20f6eab482782ed90a05d91b669b59641eaa46", + } + + +def test_stale_b12x_caps_are_rejected_before_a_gpu_launch() -> None: + @dataclass + class StaleCaps: + device: str + + def stale_bind(plan, *, mixed_qkv): + return plan, mixed_qkv + + with pytest.raises(contract.ContractError, match="omits required field"): + contract.validate_b12x_surface( + StaleCaps, + stale_bind, + SimpleNamespace(plan=object()), + required_field="kda_metadata_validation", + ) + + +def test_live_tensor_b12x_surface_requires_every_binding_input() -> None: + @dataclass + class Caps: + kda_metadata_validation: str = "transactional" + + def incomplete_bind(plan, *, mixed_qkv, output): + return plan, mixed_qkv, output + + with pytest.raises(contract.ContractError, match="omits live tensor parameters"): + contract.validate_b12x_surface( + Caps, + incomplete_bind, + SimpleNamespace(plan=object()), + required_field="kda_metadata_validation", + ) + + +def test_elf_manifest_detects_retained_native_changes(tmp_path: Path) -> None: + package = tmp_path / "vllm" + package.mkdir() + first = package / "_C_stable_libtorch.abi3.so" + second = package / "vllm-rs" + first.write_bytes(b"\x7fELFfirst") + second.write_bytes(b"\x7fELFsecond") + original = contract.elf_manifest(package) + assert [record["path"] for record in original] == [ + "_C_stable_libtorch.abi3.so", + "vllm-rs", + ] + second.write_bytes(b"\x7fELFchanged") + assert contract.canonical_sha256(contract.elf_manifest(package)) != ( + contract.canonical_sha256(original) + ) + + +def test_containerfile_reuses_vllm_and_nccl_native_artifacts() -> None: + recipe = (HERE / "Containerfile").read_text(encoding="utf-8") + assert "FROM ${PUBLIC_BASE}" in recipe + assert "COPY bundle/vllm-overlay/ /usr/local/lib/python3.12/dist-packages/" in recipe + assert "--reinstall --no-deps" in recipe + assert "record-base" in recipe and "verify-composed" in recipe + assert "native-elf-manifest-sha256" in recipe + assert "native-dispatch-manifest-sha256" in recipe + assert "setup.py bdist_wheel" not in recipe + assert "make -C /build/nccl" not in recipe + + source_copy = recipe.index( + "COPY bundle/sources/sparkcache/sparkcache /opt/sparkcache-src/sparkcache" + ) + source_verification = recipe.index('printf \'sparkcache_source_sha256=%s') + source_receipt = recipe.index("sparkcache-source-tree.sha256") + cuda_placement_copy = recipe.index( + "COPY --from=sparkcache-cuda-placement", source_verification + ) + assert source_copy < source_verification < source_receipt < cuda_placement_copy + + +def test_image_verifier_reads_the_clean_sparkcache_source_receipt() -> None: + verifier = (HERE / "verify_image.py").read_text(encoding="utf-8") + assert "sparkcache-source-tree.sha256" in verifier + assert 'artifacts["sparkcache_source_tree_sha256"]' in verifier + + +def test_build_prepares_context_below_the_temporary_workspace() -> None: + script = (HERE / "build-image.sh").read_text(encoding="utf-8") + preparer = (HERE / "prepare_context.py").read_text(encoding="utf-8") + assert 'workspace="$(mktemp -d)"' in script + assert 'context="${workspace}/context"' in script + assert 'rm -rf -- "${workspace}"' in script + assert '"core.longpaths", "true"' in preparer + + +def test_sparkcache_patches_and_contract_run_after_the_python_overlay() -> None: + recipe = (HERE / "Containerfile").read_text(encoding="utf-8") + overlay = recipe.index("COPY bundle/vllm-overlay/") + patch_020 = recipe.index("020-sparkcache-vmm-exemption.patch") + dflash_patch = recipe.index("010-dflash-draft-load-config.patch") + patch_030 = recipe.index("030-sparkcache-hma-load-failure.patch") + patch_040 = recipe.index("040-sparkcache-shared-prefix-lease.patch") + patch_041 = recipe.index("041-sparkcache-shared-prefix-attach.patch") + lease = recipe.index("verify_lease_contract.py") + assert overlay < dflash_patch < patch_020 < patch_030 < patch_040 < patch_041 < lease + + +def test_output_labels_do_not_claim_a_source_built_0b_wheel() -> None: + pins = json.loads(PINS.read_text(encoding="utf-8")) + labels = verify.expected_output_labels(pins) + assert labels["org.jovian.vllm.commit"] == pins["vllm"]["native_commit"] + assert labels["org.sparkring.vllm.python.commit"] == pins["vllm"][ + "python_commit" + ] + assert labels["org.jovian.vllm.commit"] != labels[ + "org.sparkring.vllm.python.commit" + ] + assert labels["org.sparkring.vllm.dflash-draft-loader-patch-sha256"] == ( + "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279" + ) + assert labels["org.sparkring.vllm.dflash-draft-loader-postimage-sha256"] == ( + "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4" + ) diff --git a/runtime/glm53-flash-adaptive-mtp-python-overlay/verify_image.py b/runtime/glm53-flash-adaptive-mtp-python-overlay/verify_image.py new file mode 100644 index 00000000..870bc4dc --- /dev/null +++ b/runtime/glm53-flash-adaptive-mtp-python-overlay/verify_image.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""Verify the public-base GLM-5.3 Python-overlay image.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +from pathlib import Path +from typing import Any, Iterable + + +HERE = Path(__file__).resolve().parent +PINS = HERE / "pins.json" +PINS_SCHEMA = "sparkring-glm53-public-python-overlay/v1" +RECEIPT_SCHEMA = "sparkring-glm53-public-python-overlay-image/v1" +SHA256 = re.compile(r"[0-9a-f]{64}\Z") +SHA256_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") +GIT_COMMIT = re.compile(r"[0-9a-f]{40}\Z") + + +class VerifyError(RuntimeError): + """An image differs from the public Python-overlay contract.""" + + +def run(argv: Iterable[str]) -> str: + arguments = list(argv) + completed = subprocess.run( + arguments, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if completed.returncode: + detail = completed.stderr.strip() or completed.stdout.strip() + raise VerifyError(f"command failed ({' '.join(arguments)}): {detail}") + return completed.stdout.strip() + + +def load_pins(path: Path = PINS) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict) or value.get("schema") != PINS_SCHEMA: + raise VerifyError(f"{path} does not use schema {PINS_SCHEMA}") + return value + + +def inspect_image(engine: str, image: str) -> dict[str, Any]: + values = json.loads(run((engine, "image", "inspect", image))) + if not isinstance(values, list) or len(values) != 1: + raise VerifyError("container engine returned an unexpected image inspection") + return values[0] + + +def validate_platform(document: dict[str, Any]) -> None: + if document.get("Architecture") != "arm64" or document.get("Os") != "linux": + raise VerifyError( + "image platform mismatch: expected linux/arm64, got " + f"{document.get('Os')}/{document.get('Architecture')}" + ) + if SHA256_ID.fullmatch(str(document.get("Id", ""))) is None: + raise VerifyError(f"image ID is not an immutable SHA-256: {document.get('Id')!r}") + + +def validate_labels(document: dict[str, Any], expected: dict[str, str]) -> None: + labels = document.get("Config", {}).get("Labels") or {} + if not isinstance(labels, dict): + raise VerifyError("image labels are not a JSON object") + for name, value in expected.items(): + observed = labels.get(name) + if observed != value: + raise VerifyError( + f"image label {name} mismatch: expected {value!r}, got {observed!r}" + ) + + +def validate_base(document: dict[str, Any], pins: dict[str, Any]) -> None: + validate_platform(document) + base = pins["public_base"] + if document["Id"] != base["image_id"]: + raise VerifyError( + f"public base image ID mismatch: expected {base['image_id']}, " + f"got {document['Id']}" + ) + validate_labels(document, base["labels"]) + + +def expected_output_labels(pins: dict[str, Any]) -> dict[str, str]: + labels = { + "org.opencontainers.image.base.name": pins["public_base"]["reference"], + "org.sparkring.base.image-id": pins["public_base"]["image_id"], + "org.sparkring.vllm.native.commit": pins["vllm"]["native_commit"], + "org.sparkring.vllm.python.commit": pins["vllm"]["python_commit"], + "org.sparkring.vllm.python.tree": pins["vllm"]["python_tree"], + "org.sparkring.vllm.python-overlay-manifest-sha256": pins["vllm"][ + "overlay_manifest_sha256" + ], + "org.jovian.vllm.commit": pins["vllm"]["native_commit"], + "org.jovian.b12x.commit": pins["b12x"]["commit"], + "org.sparkring.b12x.tree": pins["b12x"]["tree"], + "org.sparkring.nccl.commit": pins["public_base"]["labels"][ + "org.sparkring.nccl.commit" + ], + "org.sparkcache.source-revision": pins["sparkcache"]["commit"], + "org.sparkcache.source-tree": pins["sparkcache"]["tree"], + "org.sparkcache.source-sha256": pins["sparkcache"]["source_tree_sha256"], + "org.sparkcache.vllm-contract-sha256": pins["sparkcache"]["contract"][ + "sha256" + ], + "org.sparkcache.deployment-profile": "glm53-flash-adaptive-mtp-python-overlay", + } + runtime_patches = pins["vllm"].get("runtime_patches", ()) + if len(runtime_patches) != 1: + raise VerifyError("runtime contract requires one DFlash loader patch") + patch = runtime_patches[0] + labels["org.sparkring.vllm.dflash-draft-loader-patch-sha256"] = patch[ + "sha256" + ] + labels["org.sparkring.vllm.dflash-draft-loader-postimage-sha256"] = patch[ + "postimage_sha256" + ] + cleanup = pins.get("runtime_cleanup", {}).get("deep_ep") + if cleanup is not None: + labels["org.sparkring.runtime.removed-deep-ep-distribution"] = ( + f"{cleanup['distribution']}=={cleanup['version']}" + ) + labels["org.sparkring.runtime.deep-ep-removal-receipt-sha256"] = cleanup[ + "receipt_sha256" + ] + return labels + + +def validate_output(document: dict[str, Any], pins: dict[str, Any]) -> None: + validate_platform(document) + validate_labels(document, expected_output_labels(pins)) + labels = document.get("Config", {}).get("Labels") or {} + revision = labels.get("org.opencontainers.image.revision") + if not isinstance(revision, str) or GIT_COMMIT.fullmatch(revision) is None: + raise VerifyError("SparkRing image revision must contain 40 hexadecimal characters") + receipt = labels.get("org.sparkring.source-receipt-sha256") + if not isinstance(receipt, str) or SHA256.fullmatch(receipt) is None: + raise VerifyError("source receipt label must contain a lowercase SHA-256") + + +def runtime_contract_probe(engine: str, image: str) -> dict[str, Any]: + root = "/opt/sparkring/runtime/python-overlay" + output = run( + ( + engine, + "run", + "--rm", + "--entrypoint", + "python3", + image, + f"{root}/overlay_contract.py", + "--pins", + f"{root}/pins.json", + "--manifest", + f"{root}/vllm-python-overlay.json", + "verify-composed", + "--root", + "/usr/local/lib/python3.12/dist-packages", + "--site-root", + "/usr/local/lib/python3.12/dist-packages", + "--console-script", + "/usr/local/bin/vllm", + "--base-record", + f"{root}/retained-native.json", + ) + ) + try: + return json.loads(output.splitlines()[-1]) + except (IndexError, json.JSONDecodeError) as exc: + raise VerifyError(f"runtime contract probe did not return JSON: {output!r}") from exc + + +def artifact_probe(engine: str, image: str) -> dict[str, Any]: + program = r""" +import hashlib +import importlib.metadata +import importlib.util +import json +import pathlib + +root = pathlib.Path('/opt/sparkring/runtime/python-overlay') +cuda_placement = pathlib.Path('/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so') +contract = pathlib.Path('/opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json') +sparkcache_source_receipt = root / 'sparkcache-source-tree.sha256' +wheel_receipt = (root / 'b12x-wheel.sha256').read_text(encoding='utf-8').split() +deep_ep_receipt = root / 'deep-ep-removal-receipt.json' +try: + importlib.metadata.distribution('deep_ep') + deep_ep_distribution_present = True +except importlib.metadata.PackageNotFoundError: + deep_ep_distribution_present = False +print(json.dumps({ + 'b12x_wheel_sha256': wheel_receipt[0], + 'b12x_wheel': pathlib.Path(wheel_receipt[1]).name, + 'sparkcache_cuda_placement_sha256': hashlib.sha256(cuda_placement.read_bytes()).hexdigest(), + 'sparkcache_contract_sha256': hashlib.sha256(contract.read_bytes()).hexdigest(), + 'sparkcache_source_tree_sha256': sparkcache_source_receipt.read_text(encoding='utf-8').strip(), + 'source_receipt_sha256': hashlib.sha256((root / 'source-receipt.json').read_bytes()).hexdigest(), + 'deep_ep_receipt': json.loads(deep_ep_receipt.read_text(encoding='utf-8')) if deep_ep_receipt.is_file() else None, + 'deep_ep_receipt_sha256': hashlib.sha256(deep_ep_receipt.read_bytes()).hexdigest() if deep_ep_receipt.is_file() else None, + 'deep_ep_module_present': importlib.util.find_spec('deep_ep') is not None, + 'deep_ep_owners': importlib.metadata.packages_distributions().get('deep_ep') or [], + 'deep_ep_distribution_present': deep_ep_distribution_present, +}, sort_keys=True)) +""".strip() + output = run( + ( + engine, + "run", + "--rm", + "--entrypoint", + "python3", + image, + "-c", + program, + ) + ) + try: + return json.loads(output) + except json.JSONDecodeError as exc: + raise VerifyError(f"artifact probe did not return JSON: {output!r}") from exc + + +def verify_runtime_cleanup(artifacts: dict[str, Any], pins: dict[str, Any]) -> None: + cleanup = pins.get("runtime_cleanup", {}).get("deep_ep") + if cleanup is None: + return + expected_removal_receipt = { + "schema": "sparkring-python-distribution-removal/v1", + "status": "implemented", + "module": cleanup["module"], + "distribution": cleanup["distribution"], + "version": cleanup["version"], + "postcondition": "module-absent", + "reason": cleanup["reason"], + } + if artifacts.get("deep_ep_receipt") != expected_removal_receipt: + raise VerifyError("installed DeepEP removal receipt differs from its pin") + if artifacts.get("deep_ep_receipt_sha256") != cleanup["receipt_sha256"]: + raise VerifyError("installed DeepEP removal receipt checksum differs") + if artifacts.get("deep_ep_module_present") is not False: + raise VerifyError("deep_ep remains importable in the composed image") + if artifacts.get("deep_ep_owners") != []: + raise VerifyError("deep_ep still has installed distribution owners") + if artifacts.get("deep_ep_distribution_present") is not False: + raise VerifyError("the attested deep_ep distribution remains installed") + + +def verify_image(engine: str, image: str, pins_path: Path = PINS) -> dict[str, Any]: + pins = load_pins(pins_path) + inspection = inspect_image(engine, image) + validate_output(inspection, pins) + runtime = runtime_contract_probe(engine, image) + artifacts = artifact_probe(engine, image) + if runtime.get("vllm_python_files_verified") != 31: + raise VerifyError("runtime did not verify all 31 vLLM Python overlay files") + expected_runtime_patches = [ + { + "path": record["target"], + "sha256": record["postimage_sha256"], + } + for record in pins["vllm"].get("runtime_patches", ()) + ] + if runtime.get("vllm_runtime_patches") != expected_runtime_patches: + raise VerifyError("runtime did not verify the DFlash draft-loader patch") + if artifacts["sparkcache_contract_sha256"] != pins["sparkcache"]["contract"]["sha256"]: + raise VerifyError("installed SparkCache lease contract differs from its pin") + if artifacts["sparkcache_source_tree_sha256"] != pins["sparkcache"][ + "source_tree_sha256" + ]: + raise VerifyError("clean SparkCache source receipt differs from its pin") + verify_runtime_cleanup(artifacts, pins) + for name in ( + "b12x_wheel_sha256", + "sparkcache_cuda_placement_sha256", + "source_receipt_sha256", + ): + if SHA256.fullmatch(str(artifacts.get(name, ""))) is None: + raise VerifyError(f"artifact probe returned an invalid {name}") + labels = inspection.get("Config", {}).get("Labels") or {} + dynamic_labels = { + "org.sparkring.vllm.native-elf-manifest-sha256": runtime[ + "native_elf_manifest_sha256" + ], + "org.sparkring.vllm.native-dispatch-manifest-sha256": runtime[ + "native_dispatch_manifest_sha256" + ], + "org.sparkcache.cuda-placement-library-sha256": artifacts[ + "sparkcache_cuda_placement_sha256" + ], + "org.sparkring.source-receipt-sha256": artifacts[ + "source_receipt_sha256" + ], + } + for name, expected in dynamic_labels.items(): + if labels.get(name) != expected: + raise VerifyError( + f"image label {name} mismatch: expected {expected}, got {labels.get(name)}" + ) + return { + "schema": RECEIPT_SCHEMA, + "status": "implemented", + "image": image, + "image_id": inspection["Id"], + "repo_digests": sorted(inspection.get("RepoDigests") or []), + "platform": "linux/arm64", + "labels": dict(sorted((inspection.get("Config", {}).get("Labels") or {}).items())), + "runtime_contract": runtime, + "artifacts": artifacts, + "limitation": ( + "Construction, Python imports, retained native bytes, and source contracts " + "are verified. Four-rank model loading and serving are not qualified." + ), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--engine", default="docker") + parser.add_argument("--pins", type=Path, default=PINS) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--base-image") + group.add_argument("--image") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + try: + pins = load_pins(args.pins.resolve()) + if args.base_image: + document = inspect_image(args.engine, args.base_image) + validate_base(document, pins) + result: dict[str, Any] = { + "status": "qualified", + "base_image": args.base_image, + "image_id": document["Id"], + } + else: + result = verify_image(args.engine, args.image, args.pins.resolve()) + except (OSError, KeyError, json.JSONDecodeError, VerifyError) as exc: + parser.error(str(exc)) + rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8", newline="\n") + print(rendered, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/runtime/glm53-flash-adaptive-mtp-python-overlay/vllm-python-overlay.json b/runtime/glm53-flash-adaptive-mtp-python-overlay/vllm-python-overlay.json new file mode 100644 index 00000000..3e657022 --- /dev/null +++ b/runtime/glm53-flash-adaptive-mtp-python-overlay/vllm-python-overlay.json @@ -0,0 +1,232 @@ +{ + "schema": "sparkring-vllm-python-overlay/v1", + "base": { + "repository": "https://github.com/local-inference-lab/vllm.git", + "commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "tree": "4a99033352bc1a6b00852dc0e8b1cecbdd9f0ebb" + }, + "target": { + "repository": "https://github.com/local-inference-lab/vllm.git", + "commit": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639" + }, + "files": [ + { + "path": "vllm/config/speculative.py", + "operation": "replace", + "base_sha256": "6cb3d8145f435d4bb4f6201b64df4759b2e4d46f29e4e7b3b465b1a4c8be3f2f", + "target_sha256": "fc14efbcf18ad3d67cb869591d2f480b55724d1e663bb433ee5b502976bd52b9", + "target_bytes": 84471 + }, + { + "path": "vllm/config/vllm.py", + "operation": "replace", + "base_sha256": "e6a6e4087331b897bb3116720f608cc23b43dcdbca9c65187aa54dbbd5c0e16e", + "target_sha256": "cc03756d9bebf2a128828e5fbf7e9766446884dfce016c34381bce0aa78bfd9e", + "target_bytes": 125842 + }, + { + "path": "vllm/model_executor/layers/fused_moe/b12x.py", + "operation": "replace", + "base_sha256": "ee76cc6aa801e24be8ec78a9e8502061ad48667d5ff8711e43f68fba912425bd", + "target_sha256": "34b7dc76e9a3df1cb562406dc6c304cf3abf5e234b986140656230145da1bfa6", + "target_bytes": 26664 + }, + { + "path": "vllm/model_executor/layers/logits_processor.py", + "operation": "replace", + "base_sha256": "3bf29abb2654427a1f272ae019bab966b29ab85eea2e2adcbf257e89cab48c50", + "target_sha256": "36c5a32a55ab0fbb061d9632962ccfb33322c5a24f3fb2b237bb05740f0b3ad1", + "target_bytes": 14685 + }, + { + "path": "vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py", + "operation": "replace", + "base_sha256": "310cfd16734d42a9dc28715f559e86a0f29ba3a31eed0940e296046b4841319a", + "target_sha256": "8bf8bc579dd4a80224dc1633e7513f2a0c58e07db72a736c7e41d28d3c35f3b9", + "target_bytes": 39290 + }, + { + "path": "vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py", + "operation": "replace", + "base_sha256": "8ed1dd4881983880f2d8fb360e4637d6ce71681943dbfc76891bf251972ca0a2", + "target_sha256": "908b1decb6d1aaa44416a7925f9391473b53b2b33d2f8b1e265a708d29e344a7", + "target_bytes": 90000 + }, + { + "path": "vllm/model_executor/model_loader/default_loader.py", + "operation": "replace", + "base_sha256": "9c9d54b1b650bf5affc924ebb8b8c73711e187ae7486cadd9f737bb1279cc7b8", + "target_sha256": "f391aebb45fb781e5f81ac26005a45b43d6b7190c65f34a96c091b862cd01dbb", + "target_bytes": 21047 + }, + { + "path": "vllm/model_executor/model_loader/weight_utils.py", + "operation": "replace", + "base_sha256": "b42ff2bd9301c4034592598f86a03df7d60a9fc7e13576054deb44bce9ceccf7", + "target_sha256": "5a8bd3cfc063a824e1a63b00f7409fb7679eb3769aa6af22873a5e67d1817a8b", + "target_bytes": 75236 + }, + { + "path": "vllm/models/glm5next/nvidia/model.py", + "operation": "replace", + "base_sha256": "cc07fcdc350e3d9e91cf955b43a160b347e07c9bdf2aa68d37986a4509fcda8f", + "target_sha256": "afeafe96c6022dc371c3c56060ea085f65f72582ce0787183ded3b0b0a0ae26b", + "target_bytes": 53168 + }, + { + "path": "vllm/models/glm5next/nvidia/mtp.py", + "operation": "replace", + "base_sha256": "03a601119e976168d3645f19f7c3f6420a36be5cb1b15a922340911566c990bb", + "target_sha256": "248a1472fc25530c808567110d9947b54eed736ee3fe12469f106cec70781fa1", + "target_bytes": 19005 + }, + { + "path": "vllm/models/glm5next/nvidia/pooled_indexer.py", + "operation": "replace", + "base_sha256": "69b90194146f75b44d850833737eb727c337430a886fb7acb89faadff3410e2b", + "target_sha256": "b1dfabf9146b0011427b21f035ac63720450bdbfd6ea02e38ae6bdef81817d94", + "target_bytes": 20723 + }, + { + "path": "vllm/models/qwen3_8_flash_next/model.py", + "operation": "replace", + "base_sha256": "edcfb0c4bd39332e2d55498ed293545fac17e1786e03fed40954c6c952acfe77", + "target_sha256": "c315a02c98ab9cfd889283829c94dcfc8cd6b21efc7819f39100d84cb26a09f2", + "target_bytes": 33701 + }, + { + "path": "vllm/models/qwen3_8_flash_next/mtp.py", + "operation": "replace", + "base_sha256": "5b21079bcbdb05ec85d6a1f840ce87baa0a5546dfc8308de8b3ba9fb371ce404", + "target_sha256": "8efbaca8d42097061802a7e8cedd7c84eaede0648b0b31e810e752ce5573c34c", + "target_bytes": 20071 + }, + { + "path": "vllm/models/qwen3_8_flash_next/nvidia/qsa.py", + "operation": "replace", + "base_sha256": "eafefbc18e1b2164cfd63d01cdccd0cb94770cbbe0946e3c260a476bf6f2f548", + "target_sha256": "8e21c17d382bf9b2016fbfa4195abd9c4656b203556e60c80f369cfdf0ea906d", + "target_bytes": 55541 + }, + { + "path": "vllm/models/qwen3_8_flash_next/ple_layer.py", + "operation": "replace", + "base_sha256": "dfc180223a0fba618ebf50e2777fb3e3f1bbf284284de219159391f4db7f88e9", + "target_sha256": "90f5f130d12a66a6aebd6e4c590c1c5b86fe7086ce4f502dc5ce5f8192b05d41", + "target_bytes": 40229 + }, + { + "path": "vllm/utils/b12x.py", + "operation": "replace", + "base_sha256": "caa5faa0457a53f2f4402e33df560e822039d5ffd97de27c99cec1cb2e7344fd", + "target_sha256": "0789bfc595eb69f4a9ce9cf2ebc603962ab78527eca923603d9428269f47981f", + "target_bytes": 6360 + }, + { + "path": "vllm/v1/attention/backends/mla/b12x_mla_sparse.py", + "operation": "replace", + "base_sha256": "cb6d1add482cd23dfc1f9f5847258833805b452fd9e7cfc0c84f4c5cc867e977", + "target_sha256": "5924a62c4d093f1311ed5606ab31f0fe406efc8217657badd72052f95f8189b7", + "target_bytes": 36158 + }, + { + "path": "vllm/v1/core/sched/async_scheduler.py", + "operation": "replace", + "base_sha256": "e586a0ef3c6778be56a93e7f9bb712d4de9de6e7d7fe7e3e1d51dae83ecfc508", + "target_sha256": "d58ca33d48fb1eeddd7165327ea367be88752982023d58f0f831a0b90fad2c50", + "target_bytes": 3315 + }, + { + "path": "vllm/v1/core/sched/output.py", + "operation": "replace", + "base_sha256": "62a4d3818a616c25fb2816d157162e5786134c45954eb5e7fa8c3795d0735324", + "target_sha256": "65235eba652e5a3ccee18bf3cbfeac9bf4da8fb9c61e961580f612cfb7e593bc", + "target_bytes": 12153 + }, + { + "path": "vllm/v1/core/sched/scheduler.py", + "operation": "replace", + "base_sha256": "98bfb14fa37e8c2a2db5aba96bf8a6f0e6d1c0393ef686b075208889a2e46ab3", + "target_sha256": "05c05f4b372c7a4bf76399b38b338eab657c69406b7019ab02101d2ab0c7764c", + "target_bytes": 146907 + }, + { + "path": "vllm/v1/spec_decode/dynamic/acceptance_length.py", + "operation": "add", + "base_sha256": null, + "target_sha256": "2e3167f3a16006c7de34d6eda78aae1bc0a3489171b17c41122cd782303792c8", + "target_bytes": 7298 + }, + { + "path": "vllm/v1/spec_decode/metrics.py", + "operation": "replace", + "base_sha256": "c1c6b20bbf0ae3427dc331bbf8032193cf0b1ab9f3fe552257da6e4d51f2f4f6", + "target_sha256": "1610913505ac7d87cbebb36e83deeaef9aa050ad45f698a3ed1fb2d9441ab0b3", + "target_bytes": 10872 + }, + { + "path": "vllm/v1/worker/gpu/cudagraph_utils.py", + "operation": "replace", + "base_sha256": "a6bb8f0e7e97df6892bebfcce3c3a15f264c445f205141faa80da5c3abc58dbe", + "target_sha256": "9023bca429176187e74e18026d8d25a0b57e9698bfca1dcf0485cd043dd08ebb", + "target_bytes": 42402 + }, + { + "path": "vllm/v1/worker/gpu/model_runner.py", + "operation": "replace", + "base_sha256": "be17cf4c5629f3671d40f31a1f31005fb8806f7aad8a7c25e3a0e0679b1c97a3", + "target_sha256": "764120deade163cc6e29a8b22261c09e9399f64066542d3125de927895a59134", + "target_bytes": 92890 + }, + { + "path": "vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py", + "operation": "replace", + "base_sha256": "b4d7284792d590419cefd3640c93a7a1a6e6d4674cb20e1eb393999d7acbfaf6", + "target_sha256": "851a90bf3ac3940abfc6662d6270c0d65ebe356d115f4d2343c1ba07a660c85b", + "target_bytes": 44958 + }, + { + "path": "vllm/v1/worker/gpu/spec_decode/dflash/speculator.py", + "operation": "replace", + "base_sha256": "68bae390178cb1deabc4c02c3433159bbca78d38d8ae172a8bfca9efed643a30", + "target_sha256": "2fb0f4bd41fe0662555016119c1f0e6648d7b83057b5cf14c3f5904bb3210a1a", + "target_bytes": 31894 + }, + { + "path": "vllm/v1/worker/gpu/spec_decode/extract_hidden_states.py", + "operation": "replace", + "base_sha256": "453451bedd34dea6442b50c5dff7c21315d82f30029083b32ff72e35f528dba5", + "target_sha256": "758f33d2d8b7eaddc73e48ce508dfa43313051e9d873bbc5723c1f18edf201c0", + "target_bytes": 5628 + }, + { + "path": "vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py", + "operation": "replace", + "base_sha256": "f2f99e4c6a27646737fd74733e54a328bfaff27ac34a8e3ba45e89bd5482ea05", + "target_sha256": "0dfc31a7e7f0b6247ce8ea7f580dc930828a885b28c9e4a2d4e69f7ba6b516d7", + "target_bytes": 40810 + }, + { + "path": "vllm/v1/worker/gpu/spec_decode/speculator.py", + "operation": "replace", + "base_sha256": "64407e596985e7c3928a6abbfd67f79ea4cb6477fcc0de8f279d444b40b8de08", + "target_sha256": "2a24c693204032b77bf8abe08df916ca20b119cfe60fd91ad33e87920ae37685", + "target_bytes": 17731 + }, + { + "path": "vllm/v1/worker/gpu/spec_decode/utils.py", + "operation": "replace", + "base_sha256": "39ebdfdc8de50d7fddc324aa011275dccd38f2dcc32c4e3268dbbf3ea915fe49", + "target_sha256": "be78a83c689047c84abe373650e0506e66090665f877e32ef18ddcc12da987a0", + "target_bytes": 4430 + }, + { + "path": "vllm/v1/worker/gpu_model_runner.py", + "operation": "replace", + "base_sha256": "699ecba3b6cc4a96d32621ee00d078b9dde8f4f82844ac2c5d8ec40c5e81a9e7", + "target_sha256": "5ac63bf43acc6b254282426a8a9b989cc07fcdbd07223101bdc99f91f4e4ce5f", + "target_bytes": 343829 + } + ] +} diff --git a/runtime/glm53-flash-b12x-kda-adaptive-mtp/README.md b/runtime/glm53-flash-b12x-kda-adaptive-mtp/README.md index 4b3217b0..f43ba2fb 100644 --- a/runtime/glm53-flash-b12x-kda-adaptive-mtp/README.md +++ b/runtime/glm53-flash-b12x-kda-adaptive-mtp/README.md @@ -32,8 +32,10 @@ receipt proves image construction only. Startup, semantic generation, SparkCache restore, shared-prefix concurrency, and fatal-log checks require a separate four-rank receipt. -The matching SparkCache overlay is pinned to -`FujitsuPolycom/sparkcache@20838ace3ebda570ca039cb7f1976c29da554b39`. +The matching SparkCache CUDA placement source is commit +`5d571018de5b63a9a90e5c11e6d6e86bbff4a957`, Git tree +`e864ed9ad64f771188fdb59aa9738e348134d636`, with clean deployable-source +SHA-256 `f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88`. Its Linux-byte-exact vLLM contract is `vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json`. The runtime-bound embedded-MTP identity prevents this profile from reusing e105 diff --git a/runtime/glm53-flash-b12x-kda-adaptive-mtp/pins.json b/runtime/glm53-flash-b12x-kda-adaptive-mtp/pins.json index 035670f9..8b6776dd 100644 --- a/runtime/glm53-flash-b12x-kda-adaptive-mtp/pins.json +++ b/runtime/glm53-flash-b12x-kda-adaptive-mtp/pins.json @@ -104,8 +104,9 @@ }, "sparkcache": { "repository": "https://github.com/FujitsuPolycom/sparkcache", - "commit": "20838ace3ebda570ca039cb7f1976c29da554b39", - "source_tree_sha256": "4998b24f4f504aeeb9bf92769ec720e282f546e6726d89fdfd06c4efa8d17c10", + "commit": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "tree": "e864ed9ad64f771188fdb59aa9738e348134d636", + "source_tree_sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", "lease_contract": "sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json", "lease_contract_sha256": "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024", "overlay_containerfile": "deploy/glm53_flash/Containerfile.b12x-kda-adaptive-mtp", diff --git a/runtime/glm53-flash-b12x-kda-adaptive-mtp/test_glm53_b12x_kda_adaptive_mtp_contract.py b/runtime/glm53-flash-b12x-kda-adaptive-mtp/test_glm53_b12x_kda_adaptive_mtp_contract.py index 24c2ea98..b462e9c9 100644 --- a/runtime/glm53-flash-b12x-kda-adaptive-mtp/test_glm53_b12x_kda_adaptive_mtp_contract.py +++ b/runtime/glm53-flash-b12x-kda-adaptive-mtp/test_glm53_b12x_kda_adaptive_mtp_contract.py @@ -8,9 +8,10 @@ HERE = Path(__file__).resolve().parent PINS = HERE / "pins.json" -SPARKCACHE_COMMIT = "20838ace3ebda570ca039cb7f1976c29da554b39" +SPARKCACHE_COMMIT = "5d571018de5b63a9a90e5c11e6d6e86bbff4a957" +SPARKCACHE_TREE = "e864ed9ad64f771188fdb59aa9738e348134d636" SPARKCACHE_SOURCE_SHA256 = ( - "4998b24f4f504aeeb9bf92769ec720e282f546e6726d89fdfd06c4efa8d17c10" + "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88" ) OVERLAY_CONTAINERFILE_SHA256 = ( "8e2377d034ba80b059f9a4387a6590a08e205313568ee1382e0e25342f8c5d40" @@ -49,6 +50,7 @@ def test_glm53_b12x_kda_adaptive_mtp_source_build_is_exact_and_unqualified() -> assert build["outputs"]["sparkcache_image"] is None sparkcache = pins["sparkcache"] assert sparkcache["commit"] == SPARKCACHE_COMMIT + assert sparkcache["tree"] == SPARKCACHE_TREE assert sparkcache["source_tree_sha256"] == SPARKCACHE_SOURCE_SHA256 assert ( sparkcache["overlay_containerfile_sha256"] diff --git a/runtime/glm53-flash-dflash7-python-overlay/README.md b/runtime/glm53-flash-dflash7-python-overlay/README.md new file mode 100644 index 00000000..932f40c1 --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/README.md @@ -0,0 +1,71 @@ +# GLM-5.3 DFlash7 public-base Python overlay + +Status: **implemented**, not qualified. The builder constructs and verifies an +ARM64 image but no image digest from this path has completed four-rank serving +qualification. + +The image combines these exact roles: + +- retained vLLM native extensions and wheel metadata from + `da4d7be6c97434f6942292ed8abbf4b32dc44355`; +- the 31-file vLLM Python delta at + `0b67266a0f37d6146a8403fb8482403c62f412d5`; +- B12X `b1d541f9e71a35f030d45fae437630fff7507c2a`; +- SparkCache reconstructed-page placement and bounded page-delta-read source + `5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3`, Git tree + `94c236b9dfbf5f70075eb47877fd9caaa5d8c249`, and deployable source SHA-256 + `bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63`; +- external BF16 DFlash2 weights with SHA-256 + `b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b`. + +The image builder shares the byte allowlist, retained-native verifier, exact +SparkCache patch chain, and eleven-file lease contract with +`runtime/glm53-flash-adaptive-mtp-python-overlay/`. Prepared image metadata is +rendered for external DFlash7; it does not claim adaptive MTP. + +Build on Linux ARM64: + +```bash +IMAGE='sparkring-glm53-sparkcache:dflash7-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64' \ +BUILD_RECEIPT="$PWD/glm53-dflash7-python-overlay-image-receipt.json" \ +bash runtime/glm53-flash-dflash7-python-overlay/build-image.sh +``` + +The script does not push the image. Its receipt verifies mixed vLLM +provenance, B12X, target loader dependencies, NCCL, SparkCache CUDA placement, +the clean SparkCache source receipt, the vLLM lease contract, and removal of the +unused `deep_ep==2.0.0+local` distribution. The removal receipt has SHA-256 +`65514f44829e7d176b0b2cacc9559ed22724e525b7041a8bcd4d2e02d1f372e3`. +The build accepts removal only when `deep_ep` has that one distribution owner, +then verifies that the module is absent. DFlash model files remain +operator-mounted and are verified by the runtime profile. + +Two executable profiles use external DFlash at depth seven and TP4, FP8 target +KV, 256-token vLLM blocks, 32 sequences, and SparkCache page-tail copy-on-write +publication with CUDA restore. Both are implemented but unqualified on the +composed 0b image. The conservative profile uses global safetensors. The mixed +profile uses global fastsafetensors for the target and an exact +`draft_load_config` selecting safetensors for DFlash. The image applies and +verifies the draft-loader patch before installing SparkCache patches. See +`docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md`. + +The SparkCache source accepts the canonical CUDA configuration keys directly. +No PR25 compatibility profile or legacy-key translation is required by these +profiles. + +The pinned SparkCache source from +[pull request #30](https://github.com/FujitsuPolycom/sparkcache/pull/30) +accepts canonical CUDA configuration keys, replaces a partial terminal HMA +page when the authenticated cache boundary falls inside that page, and reads +authenticated page-delta chunks with a bounded eight-worker pool while +preserving descriptor order. It does not change cache identity, page-tail wire +schemas, record geometry, vLLM patch bytes, the lease contract, or the CUDA +placement ABI. Compatible entries produced by the source at commit +`5d571018de5b63a9a90e5c11e6d6e86bbff4a957` remain in the same namespace. +Null-block publication failures remain unsupported by this source contract. +Both profiles preserve B12X compute backends and the pinned PYNCCL/NCCL +library. They disable unsupported symmetric-memory and FlashInfer all-reduce +probes, disable the all-reduce RMS fusion, select language-model-only serving, +and leave Torch thread selection unset. ModelOpt and FP8 KV warnings remain +visible because they describe supported-runtime limitations rather than unused +optional backends. diff --git a/runtime/glm53-flash-dflash7-python-overlay/build-image.sh b/runtime/glm53-flash-dflash7-python-overlay/build-image.sh new file mode 100644 index 00000000..ea79ca34 --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/build-image.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Build a GLM-5.3 SparkCache image by replacing only attested Python sources. +set -euo pipefail + +here="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(git -C "${here}" rev-parse --show-toplevel)" +pins="${here}/pins.json" +shared_overlay="${repo_root}/runtime/glm53-flash-adaptive-mtp-python-overlay" +engine="${CONTAINER_ENGINE:-docker}" +image="${IMAGE:-sparkring-glm53-sparkcache:dflash7-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64}" +receipt_path="${BUILD_RECEIPT:-${PWD}/glm53-dflash7-python-overlay-image-receipt.json}" + +fatal() { + printf 'FATAL: %s\n' "$*" >&2 + exit 78 +} + +read_pin() { + python3 - "${pins}" "$1" <<'PY' +import json +import sys + +value = json.load(open(sys.argv[1], encoding="utf-8")) +for component in sys.argv[2].split("."): + value = value[int(component)] if isinstance(value, list) else value[component] +print(value) +PY +} + +tracked_inputs=( + runtime/glm53-flash-dflash7-python-overlay + runtime/glm53-flash-adaptive-mtp-python-overlay + LICENSE +) +git -C "${repo_root}" diff --quiet HEAD -- "${tracked_inputs[@]}" || + fatal "builder inputs differ from the checked-out SparkRing revision" +untracked="$(git -C "${repo_root}" ls-files --others --exclude-standard -- "${tracked_inputs[@]}")" +[[ -z "${untracked}" ]] || + fatal "builder inputs include untracked files: ${untracked%%$'\n'*}" + +public_base="$(read_pin public_base.reference)" +public_base_id="$(read_pin public_base.image_id)" +arm_builder="$(read_pin builder.arm_builder)" +vllm_native_commit="$(read_pin vllm.native_commit)" +vllm_python_commit="$(read_pin vllm.python_commit)" +vllm_python_tree="$(read_pin vllm.python_tree)" +overlay_manifest_sha256="$(read_pin vllm.overlay_manifest_sha256)" +b12x_commit="$(read_pin b12x.commit)" +b12x_tree="$(read_pin b12x.tree)" +sparkcache_commit="$(read_pin sparkcache.commit)" +sparkcache_tree="$(read_pin sparkcache.tree)" +sparkcache_source_sha256="$(read_pin sparkcache.source_tree_sha256)" +deep_ep_removal_receipt_sha256="$(read_pin runtime_cleanup.deep_ep.receipt_sha256)" +sparkring_revision="$(git -C "${repo_root}" rev-parse HEAD)" + +"${engine}" pull --platform linux/arm64 "${public_base}" +python3 "${here}/verify_image.py" \ + --engine "${engine}" --pins "${pins}" --base-image "${public_base}" >/dev/null + +workspace="$(mktemp -d)" +context="${workspace}/context" +cleanup() { + # `workspace` is created by mktemp in this process and never accepts caller input. + rm -rf -- "${workspace}" +} +trap cleanup EXIT + +python3 "${here}/prepare_context.py" \ + --repo-root "${repo_root}" "${context}" >/dev/null +python3 "${here}/prepare_context.py" --verify "${context}" >/dev/null +source_receipt_sha256="$(sha256sum "${context}/receipt.json" | cut -d' ' -f1)" +mkdir -p "${context}/base-probe" +"${engine}" run --rm --entrypoint python3 \ + --volume "${shared_overlay}:/contract:ro" \ + --volume "${pins}:/dflash-pins.json:ro" \ + --volume "${context}/base-probe:/out" \ + "${public_base}" \ + /contract/overlay_contract.py \ + --pins /dflash-pins.json \ + --manifest /contract/vllm-python-overlay.json \ + record-base \ + --site-root /usr/local/lib/python3.12/dist-packages \ + --console-script /usr/local/bin/vllm \ + --output /out/retained-native.json >/dev/null +native_elf_manifest_sha256="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8"))["native_elf_manifest_sha256"])' "${context}/base-probe/retained-native.json")" +native_dispatch_manifest_sha256="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8"))["native_dispatch_manifest_sha256"])' "${context}/base-probe/retained-native.json")" +cuda_placement_stage="sparkring-sparkcache-cuda-placement:${sparkcache_commit:0:12}-${sparkring_revision:0:12}" +"${engine}" build \ + --platform linux/arm64 \ + --target sparkcache-cuda-placement \ + --file "${context}/bundle/runtime/Containerfile" \ + --build-arg "ARM_BUILDER=${arm_builder}" \ + --tag "${cuda_placement_stage}" \ + "${context}" +sparkcache_cuda_placement_sha256="$("${engine}" run --rm --entrypoint sha256sum \ + "${cuda_placement_stage}" \ + /build/sparkcache-cuda-placement/build-cuda/libspark_cache_placement.so | cut -d' ' -f1)" + +"${engine}" build \ + --platform linux/arm64 \ + --file "${context}/bundle/runtime/Containerfile" \ + --build-arg "PUBLIC_BASE=${public_base}" \ + --build-arg "PUBLIC_BASE_ID=${public_base_id}" \ + --build-arg "ARM_BUILDER=${arm_builder}" \ + --build-arg "VLLM_NATIVE_COMMIT=${vllm_native_commit}" \ + --build-arg "VLLM_PYTHON_COMMIT=${vllm_python_commit}" \ + --build-arg "VLLM_PYTHON_TREE=${vllm_python_tree}" \ + --build-arg "B12X_COMMIT=${b12x_commit}" \ + --build-arg "B12X_TREE=${b12x_tree}" \ + --build-arg "SPARKCACHE_COMMIT=${sparkcache_commit}" \ + --build-arg "SPARKCACHE_TREE=${sparkcache_tree}" \ + --build-arg "SPARKCACHE_SOURCE_SHA256=${sparkcache_source_sha256}" \ + --build-arg "SPARKRING_REVISION=${sparkring_revision}" \ + --build-arg "SOURCE_RECEIPT_SHA256=${source_receipt_sha256}" \ + --build-arg "OVERLAY_MANIFEST_SHA256=${overlay_manifest_sha256}" \ + --build-arg "NATIVE_ELF_MANIFEST_SHA256=${native_elf_manifest_sha256}" \ + --build-arg "NATIVE_DISPATCH_MANIFEST_SHA256=${native_dispatch_manifest_sha256}" \ + --build-arg "SPARKCACHE_CUDA_PLACEMENT_SHA256=${sparkcache_cuda_placement_sha256}" \ + --build-arg "DEEP_EP_REMOVAL_RECEIPT_SHA256=${deep_ep_removal_receipt_sha256}" \ + --tag "${image}" \ + "${context}" + +python3 "${here}/verify_image.py" \ + --engine "${engine}" --pins "${pins}" --image "${image}" \ + --output "${receipt_path}" >/dev/null +printf 'image=%s\nreceipt=%s\n' "${image}" "${receipt_path}" +"${engine}" image inspect --format '{{.Id}}' "${image}" diff --git a/runtime/glm53-flash-dflash7-python-overlay/pins.json b/runtime/glm53-flash-dflash7-python-overlay/pins.json new file mode 100644 index 00000000..a2e666a6 --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/pins.json @@ -0,0 +1,172 @@ +{ + "schema": "sparkring-glm53-public-python-overlay/v1", + "status": "implemented", + "qualification": "The DFlash7 image builder and offline verifier are implemented. The resulting image is unsupported for serving until an immutable digest passes four-rank TP4/DCP1 target and external-draft loading, semantic generation, SparkCache store and restore, failure-recovery, and concurrency checks.", + "platform": "linux/arm64", + "public_base": { + "reference": "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd", + "image_id": "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5", + "labels": { + "org.jovian.architecture": "linux-arm64-sm121", + "org.jovian.vllm.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.jovian.b12x.commit": "2fcf23a0ce269be27b2e03fece73d46e90e6aeea", + "org.jovian.transport": "sparkring-nccl-2.30.7-source-built", + "org.sparkring.nccl.commit": "73cf112295c33aee2b895f329f592f2a9b4b0f97", + "org.sparkring.nccl.patched-tree": "abdeb053b94c3f6d472cd55ae2b79ca821299009", + "org.sparkring.nccl.patch-sha256": "6709063fa1c25055ae77a9397dea5d89643f8211d25e7990bdd11597d08c0dde" + } + }, + "builder": { + "arm_builder": "pytorch/manylinuxaarch64-builder@sha256:f91599c49f526c77d01b68286f2bf943a5fd6a432d7e3f0afcc5784825908fe9", + "output_name": "sparkring-glm53-sparkcache:dflash7-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64" + }, + "vllm": { + "repository": "https://github.com/local-inference-lab/vllm.git", + "native_commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "native_tree": "4a99033352bc1a6b00852dc0e8b1cecbdd9f0ebb", + "python_commit": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "python_tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "overlay_manifest": "runtime/glm53-flash-adaptive-mtp-python-overlay/vllm-python-overlay.json", + "overlay_manifest_sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a", + "runtime_patches": [ + { + "status": "implemented", + "path": "runtime/glm53-flash-adaptive-mtp-python-overlay/patches/010-dflash-draft-load-config.patch", + "target": "vllm/v1/worker/gpu/spec_decode/dflash/utils.py", + "sha256": "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279", + "preimage_sha256": "2301c8199b73ed893dfbd3ae14ad125816f100b2d2ed034215b1f2d9aa2c23c5", + "postimage_sha256": "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4", + "contract": "DFlash passes SpeculativeConfig.draft_load_config to get_model; None retains the target LoadConfig fallback." + } + ], + "native_source_objects": { + "csrc": "9ada29088768f1bc08dadd2eed3c9738eb9ac8a1", + "cmake": "5e5bbdbe1c1b3a479656d8d6a41cc32a1982c43d", + "rust": "85c3cd52db223217d45377d3f7f884e756641de3", + "CMakeLists.txt": "bb0f51b43ef4e1c57918b551b8cf213f9059b601", + "setup.py": "ae64a13daa0f1facc255afcdb4ffcad264776b98", + "pyproject.toml": "0766645fc7481da0ec439208128b838b3348d94c", + "requirements": "d6e1c8e13cd4c4358ab422e3ef006d3f9f23e18b", + "docker/Dockerfile": "6e20f6eab482782ed90a05d91b669b59641eaa46" + }, + "retained_distribution_metadata": [ + "vllm/_version.py", + "vllm-*.dist-info", + "/usr/local/bin/vllm" + ] + }, + "b12x": { + "repository": "https://github.com/local-inference-lab/b12x.git", + "commit": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "package_version": "1.3.0", + "base_commit": "2fcf23a0ce269be27b2e03fece73d46e90e6aeea", + "base_tree": "58a046fc8faa747346f40f87166cda7e0f67ff47", + "required_caps_field": "kda_metadata_validation", + "required_caps_value": "trusted" + }, + "dependencies": { + "python": "3.12", + "torch": "2.13.0+cu130", + "cuda": "13.0.3", + "fastsafetensors": "0.3.3", + "instanttensor": "0.1.9", + "instanttensor_sdist_sha256": "d8692b97991c1a5fb2db7905b9a6ae90a7f967c7ddd853d35e41caa146750c02", + "nccl_library": "/opt/sparkring/nccl/libnccl.so.2.30.7", + "nccl_library_sha256": "5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3" + }, + "sparkcache": { + "repository": "https://github.com/FujitsuPolycom/sparkcache.git", + "commit": "5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3", + "tree": "94c236b9dfbf5f70075eb47877fd9caaa5d8c249", + "source_tree_sha256": "bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63", + "cuda_config_schema": "canonical-v1", + "canonical_cuda_config_keys": [ + "spark_cache_cuda_restore", + "spark_cache_cuda_placement_library", + "spark_cache_cuda_placement_library_sha256", + "spark_cache_cuda_placement_arena_bytes", + "spark_cache_cuda_restore_io_workers" + ], + "contract": { + "path": "sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json", + "sha256": "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024", + "files": 11 + }, + "patches": [ + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/020-sparkcache-vmm-exemption.patch", + "sha256": "370b498eebf44b4e52a2d2751fa249ad4bd3d0b6fd951b063a161fb06febbe99", + "target": "vllm/config/vllm.py", + "preimage_sha256": "cc03756d9bebf2a128828e5fbf7e9766446884dfce016c34381bce0aa78bfd9e", + "postimage_sha256": "9f64f5041f7f9d953e9f6bc53de8733b3eb4035c0753056a1f646346702a0994" + }, + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/030-sparkcache-hma-load-failure.patch", + "sha256": "0202df4b5db7bd35540eebdef51fcbe2bb01845b952964080e1fe903ee19b404", + "target": "vllm/v1/core/sched/scheduler.py", + "preimage_sha256": "05c05f4b372c7a4bf76399b38b338eab657c69406b7019ab02101d2ab0c7764c", + "postimage_sha256": "337893b6b088d12eb38d8d70c866242d085134442289b4f4574a1f162c9f11c8" + }, + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/040-sparkcache-shared-prefix-lease.patch", + "sha256": "6c6d6bdc2d6e35742ef37715e88697b1f972d23b28948690c4d8d835edcaf01b", + "target": "vllm/v1/core/kv_cache_manager.py", + "preimage_sha256": "02c71da26bbac81629248ee42b0a71bd2db817d339894d647d2ff6b66fd5ad19", + "postimage_sha256": "ee03dc9ce2b720c0be6e9f572d23580ba96eff68fe3406250557e83071654af0" + }, + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/041-sparkcache-shared-prefix-attach.patch", + "sha256": "b98e6bc06990f608fc5f0828c11b8eb453fbec0f8fbbf24ba45254810b7e67c3", + "target": "vllm/v1/core/sched/scheduler.py", + "preimage_sha256": "337893b6b088d12eb38d8d70c866242d085134442289b4f4574a1f162c9f11c8", + "postimage_sha256": "6d397c97f31e67a75efc01b5ddd89fa58db425de14fa43965ef2d6146b6b9bdb" + } + ] + }, + "runtime_cleanup": { + "deep_ep": { + "module": "deep_ep", + "distribution": "deep_ep", + "version": "2.0.0+local", + "receipt_sha256": "65514f44829e7d176b0b2cacc9559ed22724e525b7041a8bcd4d2e02d1f372e3", + "reason": "The four-rank DFlash7 profile uses B12X kernels and PYNCCL collectives. Removing unused DeepEP prevents its import probe from loading a second NCCL distribution." + } + }, + "runtime_cache_roots": { + "VLLM_CACHE_ROOT": "/cache/jit/vllm/dflash7-py-0b67266-native-da4d7be", + "B12X_CUTE_COMPILE_CACHE_DIR": "/cache/jit/b12x/b1d541f9/dflash7-cute", + "TRITON_CACHE_DIR": "/cache/jit/triton/dflash7-vllm-py-0b67266-native-da4d7be" + }, + "outputs": { + "image": null, + "image_id": null, + "image_digest": null, + "native_elf_manifest_sha256": null, + "native_dispatch_manifest_sha256": null, + "b12x_wheel_sha256": null + }, + "workload": { + "role": "external-dflash7", + "target": { + "repository": "local-inference-lab/GLM-5.3-Flash-NVFP4", + "revision": "520de24eabf507659eaef7c70f14fd584527facc", + "cache_identity_sha256": "a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9" + }, + "draft": { + "repository": "incoai/GLM-5.3-Flash-DFlash2", + "revision": "dc77ff1c99eeb2df044ee3d4f0094eb033fee410", + "config_sha256": "c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573", + "weights_sha256": "b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b", + "speculative_tokens": 7, + "tensor_parallel_size": 4 + }, + "target_loaders": { + "safetensors": "implemented", + "fastsafetensors": "implemented" + }, + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 32 + } +} diff --git a/runtime/glm53-flash-dflash7-python-overlay/prepare_context.py b/runtime/glm53-flash-dflash7-python-overlay/prepare_context.py new file mode 100644 index 00000000..a094ee79 --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/prepare_context.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Prepare the exact shared Python overlay with DFlash7 image metadata.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import shutil +from pathlib import Path +from typing import Any + + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +COMMON = ROOT / "runtime" / "glm53-flash-adaptive-mtp-python-overlay" +PINS = HERE / "pins.json" +RECEIPT_SCHEMA = "sparkring-glm53-public-python-overlay-context/v1" +DEEP_EP_RECEIPT = "deep-ep-removal-receipt.json" + + +class PrepareError(RuntimeError): + """The DFlash7 prepared context differs from its exact source contract.""" + + +def _module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise PrepareError(f"cannot load shared overlay module: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +common = _module("glm53_shared_python_overlay_prepare", COMMON / "prepare_context.py") + + +def _longpath_run(argv, *, cwd=None): + """Enable long paths in prepared Git repositories on Windows hosts.""" + + arguments = tuple(argv) + result = _original_run(arguments, cwd=cwd) + if len(arguments) >= 4 and arguments[:3] == ("git", "init", "--quiet"): + _original_run( + ("git", "-C", arguments[3], "config", "core.longpaths", "true") + ) + return result + + +_original_run = common.run +_original_copy_file = common.copy_file + + +def _routed_copy_file(source: Path, destination: Path) -> None: + """Place the DFlash7 pins in the shared prepared-context topology.""" + + if source == COMMON / "pins.json": + source = PINS + _original_copy_file(source, destination) + + +def _render_containerfile() -> str: + text = (COMMON / "Containerfile").read_text(encoding="utf-8") + pins = json.loads(PINS.read_text(encoding="utf-8")) + cleanup = pins["runtime_cleanup"]["deep_ep"] + replacements = { + "/cache/jit/vllm/py-0b67266-native-da4d7be": ( + "/cache/jit/vllm/dflash7-py-0b67266-native-da4d7be" + ), + "/cache/jit/b12x/b1d541f9/cute": ( + "/cache/jit/b12x/b1d541f9/dflash7-cute" + ), + "/cache/jit/triton/vllm-py-0b67266-native-da4d7be": ( + "/cache/jit/triton/dflash7-vllm-py-0b67266-native-da4d7be" + ), + "SparkRing GLM-5.3 adaptive-MTP Python overlay": ( + "SparkRing GLM-5.3 DFlash7 Python overlay" + ), + ( + "vLLM 0b67266 Python over da4d7be native extensions with B12X " + "b1d541f and SparkCache" + ): ( + "GLM-5.3 DFlash7 with vLLM 0b67266 Python over da4d7be native " + "extensions, B12X b1d541f, and SparkCache" + ), + "glm53-flash-adaptive-mtp-python-overlay": ( + "glm53-flash-dflash7-python-overlay" + ), + } + for old, new in replacements.items(): + if old not in text: + raise PrepareError(f"shared Containerfile omits required text: {old}") + text = text.replace(old, new) + deployment = ( + 'org.sparkcache.deployment-profile="glm53-flash-dflash7-python-overlay"' + ) + if deployment not in text: + raise PrepareError("rendered Containerfile omits the DFlash7 deployment label") + text = text.replace( + deployment, + 'org.sparkcache.cuda-config-schema="canonical-v1" \\\n ' + deployment, + ) + argument_marker = "ARG SPARKCACHE_CUDA_PLACEMENT_SHA256\n" + if text.count(argument_marker) != 1: + raise PrepareError("shared Containerfile omits the CUDA placement argument") + text = text.replace( + argument_marker, + argument_marker + "ARG DEEP_EP_REMOVAL_RECEIPT_SHA256\n", + ) + receipt_marker = ( + "COPY receipt.json ${PYTHON_OVERLAY_ROOT}/source-receipt.json\n" + ) + if text.count(receipt_marker) != 1: + raise PrepareError("shared Containerfile omits the source receipt copy") + cleanup_block = f"""{receipt_marker}COPY bundle/runtime/remove_distribution.py ${{PYTHON_OVERLAY_ROOT}}/remove_distribution.py +COPY bundle/runtime/{DEEP_EP_RECEIPT} ${{PYTHON_OVERLAY_ROOT}}/{DEEP_EP_RECEIPT} +RUN test \"$(sha256sum \"${{PYTHON_OVERLAY_ROOT}}/{DEEP_EP_RECEIPT}\" | cut -d' ' -f1)\" = \"${{DEEP_EP_REMOVAL_RECEIPT_SHA256}}\" \\ + && python3 \"${{PYTHON_OVERLAY_ROOT}}/remove_distribution.py\" \\ + --receipt \"${{PYTHON_OVERLAY_ROOT}}/{DEEP_EP_RECEIPT}\" \\ + && python3 -c 'import importlib.util; assert importlib.util.find_spec("deep_ep") is None' +""" + text = text.replace(receipt_marker, cleanup_block) + label_marker = ( + " org.sparkcache.deployment-profile=" + '"glm53-flash-dflash7-python-overlay" \\\n' + ) + if text.count(label_marker) != 1: + raise PrepareError("rendered Containerfile omits the DFlash7 profile label") + cleanup_labels = ( + label_marker + + " org.sparkring.runtime.removed-deep-ep-distribution=" + + f'"{cleanup["distribution"]}=={cleanup["version"]}" \\\n' + + " org.sparkring.runtime.deep-ep-removal-receipt-sha256=" + + '"${DEEP_EP_REMOVAL_RECEIPT_SHA256}" \\\n' + ) + text = text.replace(label_marker, cleanup_labels) + return text + + +def _render_verify_image() -> str: + text = (COMMON / "verify_image.py").read_text(encoding="utf-8") + replacements = { + "sparkring-glm53-public-python-overlay-image/v1": ( + "sparkring-glm53-dflash7-python-overlay-image/v1" + ), + "glm53-flash-adaptive-mtp-python-overlay": ( + "glm53-flash-dflash7-python-overlay" + ), + } + for old, new in replacements.items(): + if old not in text: + raise PrepareError(f"shared image verifier omits required text: {old}") + text = text.replace(old, new) + deployment = ( + '"org.sparkcache.deployment-profile": ' + '"glm53-flash-dflash7-python-overlay",' + ) + if deployment not in text: + raise PrepareError("rendered verifier omits the DFlash7 deployment label") + text = text.replace( + deployment, + '"org.sparkcache.cuda-config-schema": "canonical-v1",\n' + f" {deployment}", + ) + return text + + +def _deep_ep_removal_receipt(pins: dict[str, Any]) -> dict[str, str]: + cleanup = pins["runtime_cleanup"]["deep_ep"] + return { + "schema": "sparkring-python-distribution-removal/v1", + "status": "implemented", + "module": cleanup["module"], + "distribution": cleanup["distribution"], + "version": cleanup["version"], + "postcondition": "module-absent", + "reason": cleanup["reason"], + } + + +def _replace_runtime_files(context: Path) -> None: + runtime = context / "bundle" / "runtime" + pins = json.loads(PINS.read_text(encoding="utf-8")) + shutil.copy2(PINS, runtime / "pins.json") + shutil.copy2(HERE / "build-image.sh", runtime / "build-image.sh") + shutil.copy2(HERE / "README.md", runtime / "README.md") + shutil.copy2(HERE / "remove_distribution.py", runtime / "remove_distribution.py") + removal_receipt = runtime / DEEP_EP_RECEIPT + removal_receipt.write_text( + json.dumps(_deep_ep_removal_receipt(pins), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + if common.sha256_file(removal_receipt) != pins["runtime_cleanup"]["deep_ep"][ + "receipt_sha256" + ]: + raise PrepareError("DeepEP removal receipt differs from its pin") + (runtime / "Containerfile").write_text( + _render_containerfile(), encoding="utf-8", newline="\n" + ) + (runtime / "verify_image.py").write_text( + _render_verify_image(), encoding="utf-8", newline="\n" + ) + receipt_path = context / "receipt.json" + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + receipt["workload"] = json.loads(PINS.read_text(encoding="utf-8"))["workload"] + for name in ( + "pins.json", + "verify_image.py", + "Containerfile", + "build-image.sh", + "README.md", + "remove_distribution.py", + DEEP_EP_RECEIPT, + ): + relative = f"bundle/runtime/{name}" + receipt["files"][relative] = common.sha256_file(context / relative) + receipt_path.write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + + +def prepare(output: Path, *, repository_root: Path = ROOT) -> dict[str, Any]: + original_pins = common.PINS + original_run = common.run + original_copy_file = common.copy_file + try: + common.PINS = PINS + common.run = _longpath_run + common.copy_file = _routed_copy_file + common.prepare(output, repository_root=repository_root) + _replace_runtime_files(output) + common.verify_context(output) + return json.loads((output / "receipt.json").read_text(encoding="utf-8")) + finally: + common.PINS = original_pins + common.run = original_run + common.copy_file = original_copy_file + + +def verify_context(context: Path) -> dict[str, Any]: + original_pins = common.PINS + try: + common.PINS = PINS + return common.verify_context(context) + finally: + common.PINS = original_pins + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--repo-root", type=Path, default=ROOT) + parser.add_argument("--verify", action="store_true") + args = parser.parse_args() + try: + result = ( + verify_context(args.output.resolve()) + if args.verify + else prepare(args.output.resolve(), repository_root=args.repo_root.resolve()) + ) + except (OSError, KeyError, json.JSONDecodeError, PrepareError, common.PrepareError) as exc: + parser.error(str(exc)) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/runtime/glm53-flash-dflash7-python-overlay/remove_distribution.py b/runtime/glm53-flash-dflash7-python-overlay/remove_distribution.py new file mode 100644 index 00000000..55513580 --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/remove_distribution.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Remove one exact Python distribution after proving module ownership.""" + +from __future__ import annotations + +import argparse +import importlib +import importlib.metadata +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + + +SCHEMA = "sparkring-python-distribution-removal/v1" + + +class RemovalError(RuntimeError): + """The installed module ownership or removal result differs from the receipt.""" + + +def load_receipt(path: Path) -> dict[str, str]: + document = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(document, dict) or document.get("schema") != SCHEMA: + raise RemovalError(f"removal receipt must use schema {SCHEMA}") + expected = { + "schema", + "status", + "module", + "distribution", + "version", + "postcondition", + "reason", + } + if set(document) != expected: + raise RemovalError("removal receipt fields differ from the contract") + if document.get("status") != "implemented": + raise RemovalError("removal receipt status must be implemented") + if document.get("postcondition") != "module-absent": + raise RemovalError("removal receipt postcondition must be module-absent") + for name in ("module", "distribution", "version", "reason"): + if not isinstance(document.get(name), str) or not document[name]: + raise RemovalError(f"removal receipt {name} must be a non-empty string") + return document + + +def verify_unique_owner(receipt: dict[str, str]) -> None: + module = receipt["module"] + distribution = receipt["distribution"] + owners = importlib.metadata.packages_distributions().get(module) or [] + if owners != [distribution]: + raise RemovalError( + f"module {module} must have exactly one owner {distribution}; got {owners}" + ) + installed = importlib.metadata.distribution(distribution) + observed_name = installed.metadata.get("Name") + if observed_name != distribution or installed.version != receipt["version"]: + raise RemovalError( + f"distribution identity differs: expected {distribution}=={receipt['version']}, " + f"got {observed_name}=={installed.version}" + ) + if importlib.util.find_spec(module) is None: + raise RemovalError(f"owned module is not importable before removal: {module}") + + +def verify_absent(receipt: dict[str, str]) -> None: + importlib.invalidate_caches() + module = receipt["module"] + distribution = receipt["distribution"] + if importlib.util.find_spec(module) is not None: + raise RemovalError(f"module remains importable after removal: {module}") + owners = importlib.metadata.packages_distributions().get(module) or [] + if owners: + raise RemovalError(f"module ownership remains after removal: {owners}") + try: + importlib.metadata.distribution(distribution) + except importlib.metadata.PackageNotFoundError: + return + raise RemovalError(f"distribution metadata remains after removal: {distribution}") + + +def remove_distribution(receipt: dict[str, str]) -> dict[str, Any]: + verify_unique_owner(receipt) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "uninstall", + "--yes", + receipt["distribution"], + ], + check=True, + ) + verify_absent(receipt) + return { + "schema": SCHEMA, + "module": receipt["module"], + "distribution": receipt["distribution"], + "version": receipt["version"], + "postcondition": "module-absent", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--receipt", type=Path, required=True) + args = parser.parse_args() + try: + result = remove_distribution(load_receipt(args.receipt)) + except ( + OSError, + json.JSONDecodeError, + RemovalError, + subprocess.CalledProcessError, + ) as exc: + parser.error(str(exc)) + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/runtime/glm53-flash-dflash7-python-overlay/test_dflash7_python_overlay.py b/runtime/glm53-flash-dflash7-python-overlay/test_dflash7_python_overlay.py new file mode 100644 index 00000000..7c842b05 --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/test_dflash7_python_overlay.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +import importlib.util +import hashlib +import json +import subprocess +from pathlib import Path + +import pytest + + +HERE = Path(__file__).resolve().parent +PINS = HERE / "pins.json" + + +def _module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +prepare = _module("glm53_dflash7_overlay_prepare", HERE / "prepare_context.py") +verify = _module("glm53_dflash7_overlay_verify", HERE / "verify_image.py") +removal = _module("glm53_dflash7_remove_distribution", HERE / "remove_distribution.py") + + +def test_pins_bind_the_exact_dflash7_composition() -> None: + pins = json.loads(PINS.read_text(encoding="utf-8")) + assert pins["vllm"]["native_commit"] == ( + "da4d7be6c97434f6942292ed8abbf4b32dc44355" + ) + assert pins["vllm"]["python_commit"] == ( + "0b67266a0f37d6146a8403fb8482403c62f412d5" + ) + assert pins["b12x"]["commit"] == ( + "b1d541f9e71a35f030d45fae437630fff7507c2a" + ) + assert pins["sparkcache"]["commit"] == ( + "5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3" + ) + assert pins["sparkcache"]["tree"] == ( + "94c236b9dfbf5f70075eb47877fd9caaa5d8c249" + ) + assert pins["sparkcache"]["source_tree_sha256"] == ( + "bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63" + ) + assert pins["sparkcache"]["cuda_config_schema"] == "canonical-v1" + assert pins["sparkcache"]["canonical_cuda_config_keys"] == [ + "spark_cache_cuda_restore", + "spark_cache_cuda_placement_library", + "spark_cache_cuda_placement_library_sha256", + "spark_cache_cuda_placement_arena_bytes", + "spark_cache_cuda_restore_io_workers", + ] + workload = pins["workload"] + assert workload["role"] == "external-dflash7" + assert workload["draft"]["weights_sha256"] == ( + "b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b" + ) + assert workload["draft"]["speculative_tokens"] == 7 + assert workload["draft"]["tensor_parallel_size"] == 4 + assert workload["target_loaders"] == { + "safetensors": "implemented", + "fastsafetensors": "implemented", + } + cleanup = pins["runtime_cleanup"]["deep_ep"] + assert cleanup["distribution"] == "deep_ep" + assert cleanup["version"] == "2.0.0+local" + assert cleanup["receipt_sha256"] == ( + "65514f44829e7d176b0b2cacc9559ed22724e525b7041a8bcd4d2e02d1f372e3" + ) + patch = pins["vllm"]["runtime_patches"][0] + assert patch["sha256"] == ( + "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279" + ) + assert patch["postimage_sha256"] == ( + "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4" + ) + + +def test_rendered_image_metadata_names_dflash7_not_adaptive_mtp() -> None: + recipe = prepare._render_containerfile() + assert "SparkRing GLM-5.3 DFlash7 Python overlay" in recipe + assert "glm53-flash-dflash7-python-overlay" in recipe + assert 'org.sparkcache.cuda-config-schema="canonical-v1"' in recipe + assert "010-dflash-draft-load-config.patch" in recipe + assert "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4" in recipe + assert "remove_distribution.py" in recipe + assert "deep-ep-removal-receipt.json" in recipe + assert "DEEP_EP_REMOVAL_RECEIPT_SHA256" in recipe + assert 'find_spec("deep_ep") is None' in recipe + label_section = recipe[recipe.index("LABEL org.opencontainers.image.title=") :] + assert "adaptive-MTP" not in label_section + verifier = prepare._render_verify_image() + assert "sparkring-glm53-dflash7-python-overlay-image/v1" in verifier + assert '"glm53-flash-dflash7-python-overlay"' in verifier + + +def test_verifier_requires_the_dflash7_deployment_label() -> None: + pins = verify.shared.load_pins(PINS) + labels = verify.expected_output_labels(pins) + assert labels["org.sparkcache.deployment-profile"] == ( + "glm53-flash-dflash7-python-overlay" + ) + assert labels["org.sparkcache.cuda-config-schema"] == "canonical-v1" + assert labels["org.sparkcache.source-revision"] == ( + "5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3" + ) + assert labels["org.sparkcache.source-tree"] == ( + "94c236b9dfbf5f70075eb47877fd9caaa5d8c249" + ) + assert labels["org.sparkcache.source-sha256"] == ( + "bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63" + ) + assert labels["org.jovian.vllm.commit"] != labels[ + "org.sparkring.vllm.python.commit" + ] + assert labels["org.sparkring.vllm.dflash-draft-loader-patch-sha256"] == ( + "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279" + ) + assert labels["org.sparkring.runtime.removed-deep-ep-distribution"] == ( + "deep_ep==2.0.0+local" + ) + assert labels["org.sparkring.runtime.deep-ep-removal-receipt-sha256"] == ( + "65514f44829e7d176b0b2cacc9559ed22724e525b7041a8bcd4d2e02d1f372e3" + ) + + +def test_builder_uses_the_dflash7_runtime_path_and_receipt() -> None: + script = (HERE / "build-image.sh").read_text(encoding="utf-8") + assert "runtime/glm53-flash-dflash7-python-overlay" in script + assert "dflash7-vllm-python-0b67266-native-da4d7be" in script + assert "glm53-dflash7-python-overlay-image-receipt.json" in script + assert "dflash7" in script.lower() + assert "runtime_cleanup.deep_ep.receipt_sha256" in script + assert "DEEP_EP_REMOVAL_RECEIPT_SHA256" in script + + +def test_removal_receipt_is_content_addressed() -> None: + pins = json.loads(PINS.read_text(encoding="utf-8")) + receipt = prepare._deep_ep_removal_receipt(pins) + encoded = (json.dumps(receipt, indent=2, sort_keys=True) + "\n").encode() + + assert hashlib.sha256(encoded).hexdigest() == ( + pins["runtime_cleanup"]["deep_ep"]["receipt_sha256"] + ) + assert receipt["postcondition"] == "module-absent" + + +def test_prepared_context_records_removal_inputs(tmp_path: Path) -> None: + runtime = tmp_path / "bundle" / "runtime" + runtime.mkdir(parents=True) + (tmp_path / "receipt.json").write_text( + json.dumps({"files": {}, "workload": {}}), encoding="utf-8" + ) + + prepare._replace_runtime_files(tmp_path) + + receipt_path = runtime / prepare.DEEP_EP_RECEIPT + pins = json.loads(PINS.read_text(encoding="utf-8")) + assert hashlib.sha256(receipt_path.read_bytes()).hexdigest() == ( + pins["runtime_cleanup"]["deep_ep"]["receipt_sha256"] + ) + assert removal.load_receipt(receipt_path) == prepare._deep_ep_removal_receipt(pins) + context_receipt = json.loads( + (tmp_path / "receipt.json").read_text(encoding="utf-8") + ) + assert ( + context_receipt["files"][ + f"bundle/runtime/{prepare.DEEP_EP_RECEIPT}" + ] + == pins["runtime_cleanup"]["deep_ep"]["receipt_sha256"] + ) + assert "bundle/runtime/remove_distribution.py" in context_receipt["files"] + + +def test_image_verifier_requires_distribution_and_module_absence() -> None: + pins = verify.shared.load_pins(PINS) + artifacts = { + "deep_ep_receipt": prepare._deep_ep_removal_receipt(pins), + "deep_ep_receipt_sha256": pins["runtime_cleanup"]["deep_ep"][ + "receipt_sha256" + ], + "deep_ep_module_present": False, + "deep_ep_owners": [], + "deep_ep_distribution_present": False, + } + verify.shared.verify_runtime_cleanup(artifacts, pins) + + artifacts["deep_ep_module_present"] = True + with pytest.raises(verify.shared.VerifyError, match="remains importable"): + verify.shared.verify_runtime_cleanup(artifacts, pins) + + +def test_distribution_removal_proves_owner_and_uninstalls_exact_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + receipt = { + "schema": removal.SCHEMA, + "status": "implemented", + "module": "deep_ep", + "distribution": "deep_ep", + "version": "2.0.0+local", + "postcondition": "module-absent", + "reason": "Unused by the selected collective backends.", + } + state = {"installed": True} + commands: list[tuple[list[str], bool]] = [] + + class Distribution: + version = "2.0.0+local" + metadata = {"Name": "deep_ep"} + + def packages_distributions(): + return {"deep_ep": ["deep_ep"]} if state["installed"] else {} + + def distribution(name: str): + assert name == "deep_ep" + if not state["installed"]: + raise removal.importlib.metadata.PackageNotFoundError(name) + return Distribution() + + def find_spec(name: str): + assert name == "deep_ep" + return object() if state["installed"] else None + + def run(command: list[str], *, check: bool): + commands.append((command, check)) + state["installed"] = False + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr( + removal.importlib.metadata, "packages_distributions", packages_distributions + ) + monkeypatch.setattr(removal.importlib.metadata, "distribution", distribution) + monkeypatch.setattr(removal.importlib.util, "find_spec", find_spec) + monkeypatch.setattr(removal.subprocess, "run", run) + + result = removal.remove_distribution(receipt) + + assert commands == [ + ( + [ + removal.sys.executable, + "-m", + "pip", + "uninstall", + "--yes", + "deep_ep", + ], + True, + ) + ] + assert result["postcondition"] == "module-absent" + + +def test_distribution_removal_rejects_ambiguous_ownership( + monkeypatch: pytest.MonkeyPatch, +) -> None: + receipt = { + "module": "deep_ep", + "distribution": "deep_ep", + "version": "2.0.0+local", + } + monkeypatch.setattr( + removal.importlib.metadata, + "packages_distributions", + lambda: {"deep_ep": ["deep_ep", "another-owner"]}, + ) + + with pytest.raises(removal.RemovalError, match="exactly one owner"): + removal.verify_unique_owner(receipt) diff --git a/runtime/glm53-flash-dflash7-python-overlay/verify_image.py b/runtime/glm53-flash-dflash7-python-overlay/verify_image.py new file mode 100644 index 00000000..85b8b01a --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/verify_image.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Verify a GLM-5.3 DFlash7 Python-overlay image and its exact labels.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + + +HERE = Path(__file__).resolve().parent +COMMON = HERE.parent / "glm53-flash-adaptive-mtp-python-overlay" + + +def _module(): + path = COMMON / "verify_image.py" + spec = importlib.util.spec_from_file_location("glm53_shared_overlay_verify", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load shared image verifier: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +shared = _module() +shared.PINS = HERE / "pins.json" +shared.RECEIPT_SCHEMA = "sparkring-glm53-dflash7-python-overlay-image/v1" +_shared_expected_output_labels = shared.expected_output_labels + + +def expected_output_labels(pins: dict[str, Any]) -> dict[str, str]: + labels = _shared_expected_output_labels(pins) + labels["org.sparkcache.deployment-profile"] = ( + "glm53-flash-dflash7-python-overlay" + ) + labels["org.sparkcache.cuda-config-schema"] = "canonical-v1" + return labels + + +shared.expected_output_labels = expected_output_labels + + +def main() -> int: + return shared.main() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/config/README.md b/scripts/config/README.md index 76291c21..4b62783d 100644 --- a/scripts/config/README.md +++ b/scripts/config/README.md @@ -86,13 +86,30 @@ uses these sanitized inputs: | File | Role | |---|---| | `glm53-flash-b12x-kda-adaptive-mtp-tp4-site.example.yaml` | Four-rank TP4/DCP1 site and 20 GiB FP8 KV reservation per rank | -| `glm53-flash-b12x-kda-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json` | Adaptive MTP 3→5, 32-step window, fastsafetensors queue one, native SparkCache restore | +| `glm53-flash-b12x-kda-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json` | Adaptive MTP 3→5, 32-step window, fastsafetensors queue one, SparkCache CUDA restore | Resolve both templates with `scripts/prepare_glm53_b12x_kda_adaptive_mtp_profile.py`. The resolver rejects source, contract, image, loader, adaptive-policy, and cache-identity drift. Follow the [executable quickstart](../../docs/GLM53_B12X_KDA_ADAPTIVE_MTP_SPARKCACHE_TP4_QUICKSTART.md). +## GLM-5.3 external DFlash7 Python overlay + +Status: **implemented**, not qualified. Both profiles use external DFlash at +depth seven and TP4, FP8 target KV, 256-token blocks, 32 sequences, and +SparkCache CUDA restore with page-tail copy-on-write publication. + +| File | Status | Target loader | +|---|---|---| +| `glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json` | **implemented**, not qualified | Global safetensors for target and draft | +| `glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json` | **implemented**, not qualified | Target fastsafetensors queue one with explicit draft safetensors | + +Resolve either profile with +`scripts/prepare_glm53_dflash7_python_overlay_profile.py`. Follow the +[DFlash7 Python-overlay quickstart](../../docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md). +Both templates use only SparkCache PR #26 canonical CUDA keys; they do not +require a PR25 compatibility profile. + ## DeepSeek-V4-Flash-0731 **Status: implemented.** diff --git a/scripts/config/glm53-flash-b12x-kda-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json b/scripts/config/glm53-flash-b12x-kda-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json index ba05d55a..4a6d3adc 100644 --- a/scripts/config/glm53-flash-b12x-kda-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json +++ b/scripts/config/glm53-flash-b12x-kda-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json @@ -82,7 +82,7 @@ "--async-scheduling", "--enable-prefix-caching", "--cudagraph-metrics", - "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"3255539158b8a4fd199b4d97d89eb5231df3b39a5370881c16099a8059b09e44\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_native_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_native_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_native_library_sha256\":\"REPLACE_WITH_NATIVE_LIBRARY_SHA256\",\"spark_cache_native_arena_bytes\":268435456,\"spark_cache_native_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-b12x-kda-adaptive-mtp-fastsafetensors-initialization\"}}" + "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"3255539158b8a4fd199b4d97d89eb5231df3b39a5370881c16099a8059b09e44\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_cuda_placement_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_cuda_placement_library_sha256\":\"REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256\",\"spark_cache_cuda_placement_arena_bytes\":268435456,\"spark_cache_cuda_restore_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-b12x-kda-adaptive-mtp-fastsafetensors-initialization\"}}" ], "extra_volumes": [ {"host": "/REPLACE/GLM53_CACHE_HOST_ROOT", "container": "/cache/jit", "mode": "rw"} @@ -111,8 +111,9 @@ "weight_loader": "fastsafetensors", "weight_loader_queue_size": "1", "weight_loader_tp_nogds": "true", - "sparkcache_source_sha256": "4998b24f4f504aeeb9bf92769ec720e282f546e6726d89fdfd06c4efa8d17c10", - "sparkcache_source_revision": "20838ace3ebda570ca039cb7f1976c29da554b39", + "sparkcache_source_sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", + "sparkcache_source_revision": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "sparkcache_source_tree": "e864ed9ad64f771188fdb59aa9738e348134d636", "vllm_revision": "0b67266a0f37d6146a8403fb8482403c62f412d5" }, "required_image_labels": { @@ -122,10 +123,10 @@ "org.jovian.vllm.commit": "0b67266a0f37d6146a8403fb8482403c62f412d5", "org.sparkcache.deployment-profile": "glm53-flash-b12x-kda-adaptive-mtp-source-built", "org.opencontainers.image.base.name": "REPLACE_WITH_B12X_KDA_ADAPTIVE_MTP_RUNTIME_IMAGE", - "org.opencontainers.image.revision": "20838ace3ebda570ca039cb7f1976c29da554b39", + "org.opencontainers.image.revision": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", "org.sparkcache.parent-image-id": "sha256:0000000000000000000000000000000000000000000000000000000000000000", - "org.sparkcache.source-revision": "20838ace3ebda570ca039cb7f1976c29da554b39", - "org.sparkcache.source-sha256": "4998b24f4f504aeeb9bf92769ec720e282f546e6726d89fdfd06c4efa8d17c10", + "org.sparkcache.source-revision": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "org.sparkcache.source-sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", "org.sparkring.nccl.commit": "73cf112295c33aee2b895f329f592f2a9b4b0f97", "org.sparkring.nccl.patch-sha256": "6709063fa1c25055ae77a9397dea5d89643f8211d25e7990bdd11597d08c0dde", "org.sparkring.nccl.patched-tree": "abdeb053b94c3f6d472cd55ae2b79ca821299009" @@ -133,6 +134,6 @@ "attestation_hook": [ "/bin/sh", "-ec", - "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && test \"$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location(\"source_identity\", \"/opt/sparkcache-source-identity.py\"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path(\"/opt/sparkcache-src/sparkcache\")))')\" = 4998b24f4f504aeeb9bf92769ec720e282f546e6726d89fdfd06c4efa8d17c10 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json | cut -d ' ' -f1)\" = 6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024 && test \"$(sha256sum /usr/local/lib/python3.12/dist-packages/vllm/config/vllm.py | cut -d ' ' -f1)\" = 9f64f5041f7f9d953e9f6bc53de8733b3eb4035c0753056a1f646346702a0994 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_NATIVE_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json" + "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && test \"$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location(\"source_identity\", \"/opt/sparkcache-source-identity.py\"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path(\"/opt/sparkcache-src/sparkcache\")))')\" = f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json | cut -d ' ' -f1)\" = 6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024 && test \"$(sha256sum /usr/local/lib/python3.12/dist-packages/vllm/config/vllm.py | cut -d ' ' -f1)\" = 9f64f5041f7f9d953e9f6bc53de8733b3eb4035c0753056a1f646346702a0994 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json" ] } diff --git a/scripts/config/glm53-flash-dflash2-bf16-tp4-dcp1-sparkcache.example.json b/scripts/config/glm53-flash-dflash2-bf16-tp4-dcp1-sparkcache.example.json index 955191d3..e9272647 100644 --- a/scripts/config/glm53-flash-dflash2-bf16-tp4-dcp1-sparkcache.example.json +++ b/scripts/config/glm53-flash-dflash2-bf16-tp4-dcp1-sparkcache.example.json @@ -81,7 +81,7 @@ "--async-scheduling", "--enable-prefix-caching", "--cudagraph-metrics", - "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_native_restore\":false,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288}}" + "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":false,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288}}" ], "extra_volumes": [ {"host": "/REPLACE/DFLASH_MODEL_HOST_PATH", "container": "/mtp-draft", "mode": "ro"}, diff --git a/scripts/config/glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json b/scripts/config/glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json new file mode 100644 index 00000000..69e50207 --- /dev/null +++ b/scripts/config/glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json @@ -0,0 +1,201 @@ +{ + "schema": "sparkring-runtime-profile/v1", + "profile_id": "glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1", + "model_family": "glm53-flash", + "engine": "docker", + "container_name": "glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4", + "image": "REPLACE_WITH_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_IMAGE", + "image_id": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "model_host_path": "/REPLACE/TARGET_MODEL_HOST_PATH", + "model_container_path": "/models/target", + "shm_size": "32g", + "startup_timeout_seconds": 7200, + "environment": { + "B12X_CUTE_COMPILE_CACHE_DIR": "/cache/jit/b12x/b1d541f9/dflash7-cute", + "CMAKE_CUDA_ARCHITECTURES": "121", + "CUTE_DSL_ARCH": "sm_121a", + "FLASHINFER_CUDA_ARCH_LIST": "12.1f", + "HF_HUB_OFFLINE": "1", + "LD_PRELOAD": "/opt/sparkring/nccl/libnccl.so.2", + "NCCL_ALGO": "Ring", + "NCCL_CROSS_NIC": "1", + "NCCL_CUMEM_ENABLE": "0", + "NCCL_DEBUG": "WARN", + "NCCL_IB_DISABLE": "0", + "NCCL_IB_MERGE_NICS": "0", + "NCCL_IB_SUBNET_AWARE_ROUTING": "1", + "NCCL_IGNORE_CPU_AFFINITY": "1", + "NCCL_MAX_NCHANNELS": "4", + "NCCL_MIN_NCHANNELS": "4", + "NCCL_NET": "IB", + "NCCL_NET_PLUGIN": "none", + "NCCL_P2P_LEVEL": "SYS", + "NCCL_PROTO": "LL,LL128,Simple", + "NCCL_SWITCHLESS_RING_ONLY": "1", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "TORCHINDUCTOR_CACHE_DIR": "/cache/jit/torchinductor/dflash7-vllm-py-0b67266-native-da4d7be", + "TORCHINDUCTOR_COMPILE_THREADS": "1", + "TORCH_CUDA_ARCH_LIST": "12.1a", + "TRANSFORMERS_OFFLINE": "1", + "TRITON_CACHE_DIR": "/cache/jit/triton/dflash7-vllm-py-0b67266-native-da4d7be", + "VLLM_ALLREDUCE_USE_FLASHINFER": "0", + "VLLM_ALLREDUCE_USE_SYMM_MEM": "0", + "VLLM_B12X_MOE_FP4_FORCE_A16": "0", + "VLLM_CACHE_ROOT": "/cache/jit/vllm/dflash7-py-0b67266-native-da4d7be", + "VLLM_ENABLE_PCIE_ALLREDUCE": "0", + "VLLM_FASTSAFETENSORS_QUEUE_SIZE": "1", + "VLLM_NCCL_SO_PATH": "/opt/sparkring/nccl/libnccl.so.2", + "VLLM_NO_USAGE_STATS": "1", + "VLLM_PLUGINS": "", + "XDG_CACHE_HOME": "/cache/jit" + }, + "extra_vllm_args": [ + "--served-model-name", + "glm-5.3-flash-nvfp4-dflash7-python-overlay-0b67266-on-da4d7be-b12x-b1d541f-tp4", + "--host", + "0.0.0.0", + "--pipeline-parallel-size", + "1", + "--disable-custom-all-reduce", + "--mamba-cache-mode", + "align", + "--language-model-only", + "--enable-chunked-prefill", + "--dtype", + "bfloat16", + "--kv-cache-dtype", + "fp8", + "--quantization", + "modelopt_mixed", + "--attention-backend", + "B12X", + "--block-size", + "256", + "--moe-backend", + "b12x", + "--linear-backend", + "b12x", + "--no-enable-flashinfer-autotune", + "--load-format", + "fastsafetensors", + "--enable-auto-tool-choice", + "--tool-call-parser", + "glm47", + "--reasoning-parser", + "glm45", + "--kda-prefill-backend", + "triton", + "--gpu-memory-utilization", + "0.80", + "--max-num-batched-tokens", + "8192", + "--speculative-config", + "{\"method\":\"dflash\",\"model\":\"/dflash-draft\",\"num_speculative_tokens\":7,\"draft_tensor_parallel_size\":4,\"kv_cache_dtype\":\"auto\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\",\"draft_load_config\":{\"load_format\":\"safetensors\"}}", + "--compilation-config", + "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[8,16,32,64,128,256],\"custom_ops\":[\"all\"],\"pass_config\":{\"fuse_allreduce_rms\":false}}", + "--max-cudagraph-capture-size", + "256", + "--async-scheduling", + "--enable-prefix-caching", + "--cudagraph-metrics", + "--kv-transfer-config", + "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context/dflash7-py-0b67266-native-da4d7be-b12x-b1d541f-tail-cow\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_publication_schema\":\"tail-cow-v1\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_cuda_placement_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_cuda_placement_library_sha256\":\"REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256\",\"spark_cache_cuda_placement_arena_bytes\":268435456,\"spark_cache_cuda_restore_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-dflash7-python-overlay-0b67266-da4d7be-b1d541f-tail-cow-initialization\"}}" + ], + "extra_volumes": [ + { + "host": "/REPLACE/DFLASH_MODEL_HOST_PATH", + "container": "/dflash-draft", + "mode": "ro" + }, + { + "host": "/REPLACE/GLM53_DFLASH7_PYTHON_OVERLAY_CACHE_HOST_ROOT", + "container": "/cache/jit", + "mode": "rw" + } + ], + "extra_labels": { + "org.sparkring.model-profile": "glm53-flash-dflash7-python-overlay-fastsafetensors-tp4-dcp1", + "org.sparkring.external-cache": "sparkcache", + "org.sparkcache.publication-schema": "tail-cow-v1", + "org.sparkring.speculator": "external-dflash7", + "org.sparkring.qualification-status": "implemented" + }, + "init": true, + "security_opts": [ + "label=disable" + ], + "privileged": false, + "confirmation": "START_GLM53_FLASH_DFLASH7_PYTHON_OVERLAY_FASTSAFETENSORS_TP4", + "identity": { + "target_repository": "local-inference-lab/GLM-5.3-Flash-NVFP4", + "target_revision": "520de24eabf507659eaef7c70f14fd584527facc", + "target_cache_identity_sha256": "a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9", + "target_config_sha256": "676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996", + "target_weight_index_sha256": "0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb", + "speculator": "external_dflash", + "draft_repository": "incoai/GLM-5.3-Flash-DFlash2", + "draft_revision": "dc77ff1c99eeb2df044ee3d4f0094eb033fee410", + "draft_config_sha256": "c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573", + "draft_weights_sha256": "b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b", + "draft_speculative_tokens": "7", + "draft_tensor_parallel_size": "4", + "target_weight_loader": "fastsafetensors", + "target_weight_loader_queue_size": "1", + "target_weight_loader_tp_nogds": "true", + "dflash_loader_scope": "target_fastsafetensors_draft_safetensors", + "dflash_peak_gpu_memory_status": "implemented", + "dflash_serving_status": "implemented", + "kv_cache_dtype": "fp8", + "vllm_block_size": "256", + "max_num_seqs": "32", + "sparkcache_publication_schema": "tail-cow-v1", + "sparkcache_effective_publication_schema": "page-tail-cow-v1", + "sparkcache_source_sha256": "bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63", + "sparkcache_source_revision": "5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3", + "sparkcache_source_tree": "94c236b9dfbf5f70075eb47877fd9caaa5d8c249", + "vllm_native_revision": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "vllm_python_revision": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "vllm_python_tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "b12x_revision": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "b12x_tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "vllm_python_overlay_manifest_sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a", + "deep_ep_removed_distribution": "deep_ep==2.0.0+local", + "deep_ep_module_status": "absent", + "allowed_runtime_warnings": "modelopt_experimental_quantization,fp8_kv_cache_accuracy" + }, + "required_image_labels": { + "org.jovian.architecture": "linux-arm64-sm121", + "org.jovian.b12x.commit": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "org.jovian.transport": "sparkring-nccl-2.30.7-source-built", + "org.jovian.vllm.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.sparkring.vllm.native.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.sparkring.vllm.python.commit": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "org.sparkring.vllm.python.tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "org.sparkring.vllm.python-overlay-manifest-sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a", + "org.sparkring.vllm.dflash-draft-loader-patch-sha256": "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279", + "org.sparkring.vllm.dflash-draft-loader-postimage-sha256": "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4", + "org.sparkring.vllm.native-elf-manifest-sha256": "REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256", + "org.sparkring.vllm.native-dispatch-manifest-sha256": "REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256", + "org.sparkring.b12x.tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "org.opencontainers.image.base.name": "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd", + "org.sparkring.base.image-id": "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5", + "org.sparkcache.deployment-profile": "glm53-flash-dflash7-python-overlay", + "org.sparkcache.cuda-config-schema": "canonical-v1", + "org.sparkcache.cuda-placement-library-sha256": "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256", + "org.sparkcache.source-revision": "5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3", + "org.sparkcache.source-tree": "94c236b9dfbf5f70075eb47877fd9caaa5d8c249", + "org.sparkcache.source-sha256": "bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63", + "org.sparkcache.vllm-contract-sha256": "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024", + "org.sparkring.runtime.removed-deep-ep-distribution": "deep_ep==2.0.0+local", + "org.sparkring.runtime.deep-ep-removal-receipt-sha256": "65514f44829e7d176b0b2cacc9559ed22724e525b7041a8bcd4d2e02d1f372e3", + "org.sparkring.source-receipt-sha256": "REPLACE_WITH_SOURCE_RECEIPT_SHA256", + "org.sparkring.nccl.commit": "73cf112295c33aee2b895f329f592f2a9b4b0f97", + "org.sparkring.nccl.patch-sha256": "6709063fa1c25055ae77a9397dea5d89643f8211d25e7990bdd11597d08c0dde", + "org.sparkring.nccl.patched-tree": "abdeb053b94c3f6d472cd55ae2b79ca821299009" + }, + "attestation_hook": [ + "/bin/sh", + "-ec", + "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test -f /dflash-draft/config.json && test -f /dflash-draft/model.safetensors && test \"$(sha256sum /dflash-draft/config.json | cut -d ' ' -f1)\" = c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573 && test \"$(sha256sum /dflash-draft/model.safetensors | cut -d ' ' -f1)\" = b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && grep -q '\"tail-cow-v1\"' /opt/sparkcache-src/sparkcache/spark_context_cache_config.py && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/vllm-python-overlay.json | cut -d ' ' -f1)\" = e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/source-receipt.json | cut -d ' ' -f1)\" = REPLACE_WITH_SOURCE_RECEIPT_SHA256 && test \"$(python3 -c 'import json; print(json.load(open(\"/opt/sparkring/runtime/python-overlay/retained-native.json\", encoding=\"utf-8\"))[\"native_elf_manifest_sha256\"])')\" = REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256 && test \"$(python3 -c 'import json; print(json.load(open(\"/opt/sparkring/runtime/python-overlay/retained-native.json\", encoding=\"utf-8\"))[\"native_dispatch_manifest_sha256\"])')\" = REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256 && test \"$(cat /opt/sparkring/runtime/python-overlay/sparkcache-source-tree.sha256)\" = bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json | cut -d ' ' -f1)\" = 6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256 && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/deep-ep-removal-receipt.json | cut -d ' ' -f1)\" = 65514f44829e7d176b0b2cacc9559ed22724e525b7041a8bcd4d2e02d1f372e3 && python3 -c 'import importlib.util; assert importlib.util.find_spec(\"deep_ep\") is None' && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json" + ] +} diff --git a/scripts/config/glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json b/scripts/config/glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json new file mode 100644 index 00000000..fc294973 --- /dev/null +++ b/scripts/config/glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json @@ -0,0 +1,198 @@ +{ + "schema": "sparkring-runtime-profile/v1", + "profile_id": "glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1", + "model_family": "glm53-flash", + "engine": "docker", + "container_name": "glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4", + "image": "REPLACE_WITH_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_IMAGE", + "image_id": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "model_host_path": "/REPLACE/TARGET_MODEL_HOST_PATH", + "model_container_path": "/models/target", + "shm_size": "32g", + "startup_timeout_seconds": 7200, + "environment": { + "B12X_CUTE_COMPILE_CACHE_DIR": "/cache/jit/b12x/b1d541f9/dflash7-cute", + "CMAKE_CUDA_ARCHITECTURES": "121", + "CUTE_DSL_ARCH": "sm_121a", + "FLASHINFER_CUDA_ARCH_LIST": "12.1f", + "HF_HUB_OFFLINE": "1", + "LD_PRELOAD": "/opt/sparkring/nccl/libnccl.so.2", + "NCCL_ALGO": "Ring", + "NCCL_CROSS_NIC": "1", + "NCCL_CUMEM_ENABLE": "0", + "NCCL_DEBUG": "WARN", + "NCCL_IB_DISABLE": "0", + "NCCL_IB_MERGE_NICS": "0", + "NCCL_IB_SUBNET_AWARE_ROUTING": "1", + "NCCL_IGNORE_CPU_AFFINITY": "1", + "NCCL_MAX_NCHANNELS": "4", + "NCCL_MIN_NCHANNELS": "4", + "NCCL_NET": "IB", + "NCCL_NET_PLUGIN": "none", + "NCCL_P2P_LEVEL": "SYS", + "NCCL_PROTO": "LL,LL128,Simple", + "NCCL_SWITCHLESS_RING_ONLY": "1", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "TORCHINDUCTOR_CACHE_DIR": "/cache/jit/torchinductor/dflash7-vllm-py-0b67266-native-da4d7be", + "TORCHINDUCTOR_COMPILE_THREADS": "1", + "TORCH_CUDA_ARCH_LIST": "12.1a", + "TRANSFORMERS_OFFLINE": "1", + "TRITON_CACHE_DIR": "/cache/jit/triton/dflash7-vllm-py-0b67266-native-da4d7be", + "VLLM_ALLREDUCE_USE_FLASHINFER": "0", + "VLLM_ALLREDUCE_USE_SYMM_MEM": "0", + "VLLM_B12X_MOE_FP4_FORCE_A16": "0", + "VLLM_CACHE_ROOT": "/cache/jit/vllm/dflash7-py-0b67266-native-da4d7be", + "VLLM_ENABLE_PCIE_ALLREDUCE": "0", + "VLLM_NCCL_SO_PATH": "/opt/sparkring/nccl/libnccl.so.2", + "VLLM_NO_USAGE_STATS": "1", + "VLLM_PLUGINS": "", + "XDG_CACHE_HOME": "/cache/jit" + }, + "extra_vllm_args": [ + "--served-model-name", + "glm-5.3-flash-nvfp4-dflash7-python-overlay-0b67266-on-da4d7be-b12x-b1d541f-tp4", + "--host", + "0.0.0.0", + "--pipeline-parallel-size", + "1", + "--disable-custom-all-reduce", + "--mamba-cache-mode", + "align", + "--language-model-only", + "--enable-chunked-prefill", + "--dtype", + "bfloat16", + "--kv-cache-dtype", + "fp8", + "--quantization", + "modelopt_mixed", + "--attention-backend", + "B12X", + "--block-size", + "256", + "--moe-backend", + "b12x", + "--linear-backend", + "b12x", + "--no-enable-flashinfer-autotune", + "--load-format", + "safetensors", + "--enable-auto-tool-choice", + "--tool-call-parser", + "glm47", + "--reasoning-parser", + "glm45", + "--kda-prefill-backend", + "triton", + "--gpu-memory-utilization", + "0.80", + "--max-num-batched-tokens", + "8192", + "--speculative-config", + "{\"method\":\"dflash\",\"model\":\"/dflash-draft\",\"num_speculative_tokens\":7,\"draft_tensor_parallel_size\":4,\"kv_cache_dtype\":\"auto\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\"}", + "--compilation-config", + "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[8,16,32,64,128,256],\"custom_ops\":[\"all\"],\"pass_config\":{\"fuse_allreduce_rms\":false}}", + "--max-cudagraph-capture-size", + "256", + "--async-scheduling", + "--enable-prefix-caching", + "--cudagraph-metrics", + "--kv-transfer-config", + "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context/dflash7-py-0b67266-native-da4d7be-b12x-b1d541f-safetensors-tail-cow\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_publication_schema\":\"tail-cow-v1\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_cuda_placement_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_cuda_placement_library_sha256\":\"REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256\",\"spark_cache_cuda_placement_arena_bytes\":268435456,\"spark_cache_cuda_restore_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-dflash7-python-overlay-0b67266-da4d7be-b1d541f-safetensors-tail-cow-initialization\"}}" + ], + "extra_volumes": [ + { + "host": "/REPLACE/DFLASH_MODEL_HOST_PATH", + "container": "/dflash-draft", + "mode": "ro" + }, + { + "host": "/REPLACE/GLM53_DFLASH7_PYTHON_OVERLAY_CACHE_HOST_ROOT", + "container": "/cache/jit", + "mode": "rw" + } + ], + "extra_labels": { + "org.sparkring.model-profile": "glm53-flash-dflash7-python-overlay-safetensors-tp4-dcp1", + "org.sparkring.external-cache": "sparkcache", + "org.sparkcache.publication-schema": "tail-cow-v1", + "org.sparkring.speculator": "external-dflash7", + "org.sparkring.qualification-status": "implemented" + }, + "init": true, + "security_opts": [ + "label=disable" + ], + "privileged": false, + "confirmation": "START_GLM53_FLASH_DFLASH7_PYTHON_OVERLAY_SAFETENSORS_TP4", + "identity": { + "target_repository": "local-inference-lab/GLM-5.3-Flash-NVFP4", + "target_revision": "520de24eabf507659eaef7c70f14fd584527facc", + "target_cache_identity_sha256": "a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9", + "target_config_sha256": "676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996", + "target_weight_index_sha256": "0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb", + "speculator": "external_dflash", + "draft_repository": "incoai/GLM-5.3-Flash-DFlash2", + "draft_revision": "dc77ff1c99eeb2df044ee3d4f0094eb033fee410", + "draft_config_sha256": "c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573", + "draft_weights_sha256": "b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b", + "draft_speculative_tokens": "7", + "draft_tensor_parallel_size": "4", + "target_weight_loader": "safetensors", + "dflash_loader_scope": "global_load_config_reaches_target_and_draft", + "dflash_peak_gpu_memory_status": "implemented", + "dflash_serving_status": "implemented", + "kv_cache_dtype": "fp8", + "vllm_block_size": "256", + "max_num_seqs": "32", + "sparkcache_publication_schema": "tail-cow-v1", + "sparkcache_effective_publication_schema": "page-tail-cow-v1", + "sparkcache_source_sha256": "bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63", + "sparkcache_source_revision": "5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3", + "sparkcache_source_tree": "94c236b9dfbf5f70075eb47877fd9caaa5d8c249", + "vllm_native_revision": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "vllm_python_revision": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "vllm_python_tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "b12x_revision": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "b12x_tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "vllm_python_overlay_manifest_sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a", + "deep_ep_removed_distribution": "deep_ep==2.0.0+local", + "deep_ep_module_status": "absent", + "allowed_runtime_warnings": "modelopt_experimental_quantization,fp8_kv_cache_accuracy" + }, + "required_image_labels": { + "org.jovian.architecture": "linux-arm64-sm121", + "org.jovian.b12x.commit": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "org.jovian.transport": "sparkring-nccl-2.30.7-source-built", + "org.jovian.vllm.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.sparkring.vllm.native.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.sparkring.vllm.python.commit": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "org.sparkring.vllm.python.tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "org.sparkring.vllm.python-overlay-manifest-sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a", + "org.sparkring.vllm.dflash-draft-loader-patch-sha256": "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279", + "org.sparkring.vllm.dflash-draft-loader-postimage-sha256": "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4", + "org.sparkring.vllm.native-elf-manifest-sha256": "REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256", + "org.sparkring.vllm.native-dispatch-manifest-sha256": "REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256", + "org.sparkring.b12x.tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "org.opencontainers.image.base.name": "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd", + "org.sparkring.base.image-id": "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5", + "org.sparkcache.deployment-profile": "glm53-flash-dflash7-python-overlay", + "org.sparkcache.cuda-config-schema": "canonical-v1", + "org.sparkcache.cuda-placement-library-sha256": "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256", + "org.sparkcache.source-revision": "5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3", + "org.sparkcache.source-tree": "94c236b9dfbf5f70075eb47877fd9caaa5d8c249", + "org.sparkcache.source-sha256": "bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63", + "org.sparkcache.vllm-contract-sha256": "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024", + "org.sparkring.runtime.removed-deep-ep-distribution": "deep_ep==2.0.0+local", + "org.sparkring.runtime.deep-ep-removal-receipt-sha256": "65514f44829e7d176b0b2cacc9559ed22724e525b7041a8bcd4d2e02d1f372e3", + "org.sparkring.source-receipt-sha256": "REPLACE_WITH_SOURCE_RECEIPT_SHA256", + "org.sparkring.nccl.commit": "73cf112295c33aee2b895f329f592f2a9b4b0f97", + "org.sparkring.nccl.patch-sha256": "6709063fa1c25055ae77a9397dea5d89643f8211d25e7990bdd11597d08c0dde", + "org.sparkring.nccl.patched-tree": "abdeb053b94c3f6d472cd55ae2b79ca821299009" + }, + "attestation_hook": [ + "/bin/sh", + "-ec", + "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test -f /dflash-draft/config.json && test -f /dflash-draft/model.safetensors && test \"$(sha256sum /dflash-draft/config.json | cut -d ' ' -f1)\" = c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573 && test \"$(sha256sum /dflash-draft/model.safetensors | cut -d ' ' -f1)\" = b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && grep -q '\"tail-cow-v1\"' /opt/sparkcache-src/sparkcache/spark_context_cache_config.py && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/vllm-python-overlay.json | cut -d ' ' -f1)\" = e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/source-receipt.json | cut -d ' ' -f1)\" = REPLACE_WITH_SOURCE_RECEIPT_SHA256 && test \"$(python3 -c 'import json; print(json.load(open(\"/opt/sparkring/runtime/python-overlay/retained-native.json\", encoding=\"utf-8\"))[\"native_elf_manifest_sha256\"])')\" = REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256 && test \"$(python3 -c 'import json; print(json.load(open(\"/opt/sparkring/runtime/python-overlay/retained-native.json\", encoding=\"utf-8\"))[\"native_dispatch_manifest_sha256\"])')\" = REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256 && test \"$(cat /opt/sparkring/runtime/python-overlay/sparkcache-source-tree.sha256)\" = bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json | cut -d ' ' -f1)\" = 6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256 && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/deep-ep-removal-receipt.json | cut -d ' ' -f1)\" = 65514f44829e7d176b0b2cacc9559ed22724e525b7041a8bcd4d2e02d1f372e3 && python3 -c 'import importlib.util; assert importlib.util.find_spec(\"deep_ep\") is None' && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json" + ] +} diff --git a/scripts/config/glm53-flash-e10536a-dflash2-bf16-sparkcache-tp4-dcp1.example.json b/scripts/config/glm53-flash-e10536a-dflash2-bf16-sparkcache-tp4-dcp1.example.json index 75bb2d84..c0a11543 100644 --- a/scripts/config/glm53-flash-e10536a-dflash2-bf16-sparkcache-tp4-dcp1.example.json +++ b/scripts/config/glm53-flash-e10536a-dflash2-bf16-sparkcache-tp4-dcp1.example.json @@ -81,7 +81,7 @@ "--async-scheduling", "--enable-prefix-caching", "--cudagraph-metrics", - "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_native_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_native_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_native_library_sha256\":\"REPLACE_WITH_NATIVE_LIBRARY_SHA256\",\"spark_cache_native_arena_bytes\":268435456,\"spark_cache_native_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-e10536a-dflash5-initialization\"}}" + "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_cuda_placement_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_cuda_placement_library_sha256\":\"REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256\",\"spark_cache_cuda_placement_arena_bytes\":268435456,\"spark_cache_cuda_restore_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-e10536a-dflash5-initialization\"}}" ], "extra_volumes": [ {"host": "/REPLACE/DFLASH_MODEL_HOST_PATH", "container": "/mtp-draft", "mode": "ro"}, @@ -127,6 +127,6 @@ "attestation_hook": [ "/bin/sh", "-ec", - "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test -f /mtp-draft/config.json && test -f /mtp-draft/model.safetensors && test \"$(sha256sum /mtp-draft/config.json | cut -d ' ' -f1)\" = c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573 && test \"$(sha256sum /mtp-draft/model.safetensors | cut -d ' ' -f1)\" = b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && test \"$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location(\"source_identity\", \"/opt/sparkcache-source-identity.py\"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path(\"/opt/sparkcache-src/sparkcache\")))')\" = 34108fb22ba95b457bf4b357407b176dcbf3a6db6227227b21ecee045502a16f && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json | cut -d ' ' -f1)\" = 70cd4e923d049da96bcfa4a5b460e2ff5f7460881d5cfd0621607080fd70f68f && test \"$(sha256sum /usr/local/lib/python3.12/dist-packages/vllm/config/vllm.py | cut -d ' ' -f1)\" = 852c2277bd99858cf940238856e7b7f51e9011f768d3c704c930bb9deaf73f61 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_NATIVE_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json" + "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test -f /mtp-draft/config.json && test -f /mtp-draft/model.safetensors && test \"$(sha256sum /mtp-draft/config.json | cut -d ' ' -f1)\" = c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573 && test \"$(sha256sum /mtp-draft/model.safetensors | cut -d ' ' -f1)\" = b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && test \"$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location(\"source_identity\", \"/opt/sparkcache-source-identity.py\"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path(\"/opt/sparkcache-src/sparkcache\")))')\" = 34108fb22ba95b457bf4b357407b176dcbf3a6db6227227b21ecee045502a16f && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json | cut -d ' ' -f1)\" = 70cd4e923d049da96bcfa4a5b460e2ff5f7460881d5cfd0621607080fd70f68f && test \"$(sha256sum /usr/local/lib/python3.12/dist-packages/vllm/config/vllm.py | cut -d ' ' -f1)\" = 852c2277bd99858cf940238856e7b7f51e9011f768d3c704c930bb9deaf73f61 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json" ] } diff --git a/scripts/config/glm53-flash-e10536a-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json b/scripts/config/glm53-flash-e10536a-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json index 0b878fb7..9b2822f0 100644 --- a/scripts/config/glm53-flash-e10536a-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json +++ b/scripts/config/glm53-flash-e10536a-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json @@ -82,7 +82,7 @@ "--async-scheduling", "--enable-prefix-caching", "--cudagraph-metrics", - "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"915b7f71f16348ab84f138d975f3db90f3edfca69976e1b9ffcc03f21cf5eb22\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_native_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_native_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_native_library_sha256\":\"REPLACE_WITH_NATIVE_LIBRARY_SHA256\",\"spark_cache_native_arena_bytes\":268435456,\"spark_cache_native_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-e10536a-mtp5-adaptive-fastsafetensors-initialization\"}}" + "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"915b7f71f16348ab84f138d975f3db90f3edfca69976e1b9ffcc03f21cf5eb22\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_cuda_placement_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_cuda_placement_library_sha256\":\"REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256\",\"spark_cache_cuda_placement_arena_bytes\":268435456,\"spark_cache_cuda_restore_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-e10536a-mtp5-adaptive-fastsafetensors-initialization\"}}" ], "extra_volumes": [ {"host": "/REPLACE/GLM53_CACHE_HOST_ROOT", "container": "/cache/jit", "mode": "rw"} @@ -129,6 +129,6 @@ "attestation_hook": [ "/bin/sh", "-ec", - "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && test \"$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location(\"source_identity\", \"/opt/sparkcache-source-identity.py\"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path(\"/opt/sparkcache-src/sparkcache\")))')\" = 34108fb22ba95b457bf4b357407b176dcbf3a6db6227227b21ecee045502a16f && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json | cut -d ' ' -f1)\" = 70cd4e923d049da96bcfa4a5b460e2ff5f7460881d5cfd0621607080fd70f68f && test \"$(sha256sum /usr/local/lib/python3.12/dist-packages/vllm/config/vllm.py | cut -d ' ' -f1)\" = 852c2277bd99858cf940238856e7b7f51e9011f768d3c704c930bb9deaf73f61 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_NATIVE_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json" + "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && test \"$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location(\"source_identity\", \"/opt/sparkcache-source-identity.py\"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path(\"/opt/sparkcache-src/sparkcache\")))')\" = 34108fb22ba95b457bf4b357407b176dcbf3a6db6227227b21ecee045502a16f && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json | cut -d ' ' -f1)\" = 70cd4e923d049da96bcfa4a5b460e2ff5f7460881d5cfd0621607080fd70f68f && test \"$(sha256sum /usr/local/lib/python3.12/dist-packages/vllm/config/vllm.py | cut -d ' ' -f1)\" = 852c2277bd99858cf940238856e7b7f51e9011f768d3c704c930bb9deaf73f61 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json" ] } diff --git a/scripts/config/glm53-flash-e10536a-mtp5-adaptive-sparkcache-tp4-dcp1.example.json b/scripts/config/glm53-flash-e10536a-mtp5-adaptive-sparkcache-tp4-dcp1.example.json index 1b7162d8..359f9d10 100644 --- a/scripts/config/glm53-flash-e10536a-mtp5-adaptive-sparkcache-tp4-dcp1.example.json +++ b/scripts/config/glm53-flash-e10536a-mtp5-adaptive-sparkcache-tp4-dcp1.example.json @@ -81,7 +81,7 @@ "--async-scheduling", "--enable-prefix-caching", "--cudagraph-metrics", - "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"915b7f71f16348ab84f138d975f3db90f3edfca69976e1b9ffcc03f21cf5eb22\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_native_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_native_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_native_library_sha256\":\"REPLACE_WITH_NATIVE_LIBRARY_SHA256\",\"spark_cache_native_arena_bytes\":268435456,\"spark_cache_native_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-e10536a-mtp5-adaptive-initialization\"}}" + "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"915b7f71f16348ab84f138d975f3db90f3edfca69976e1b9ffcc03f21cf5eb22\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_cuda_placement_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_cuda_placement_library_sha256\":\"REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256\",\"spark_cache_cuda_placement_arena_bytes\":268435456,\"spark_cache_cuda_restore_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-e10536a-mtp5-adaptive-initialization\"}}" ], "extra_volumes": [ {"host": "/REPLACE/GLM53_CACHE_HOST_ROOT", "container": "/cache/jit", "mode": "rw"} @@ -128,6 +128,6 @@ "attestation_hook": [ "/bin/sh", "-ec", - "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && test \"$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location(\"source_identity\", \"/opt/sparkcache-source-identity.py\"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path(\"/opt/sparkcache-src/sparkcache\")))')\" = 34108fb22ba95b457bf4b357407b176dcbf3a6db6227227b21ecee045502a16f && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json | cut -d ' ' -f1)\" = 70cd4e923d049da96bcfa4a5b460e2ff5f7460881d5cfd0621607080fd70f68f && test \"$(sha256sum /usr/local/lib/python3.12/dist-packages/vllm/config/vllm.py | cut -d ' ' -f1)\" = 852c2277bd99858cf940238856e7b7f51e9011f768d3c704c930bb9deaf73f61 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_NATIVE_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json" + "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && test \"$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location(\"source_identity\", \"/opt/sparkcache-source-identity.py\"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path(\"/opt/sparkcache-src/sparkcache\")))')\" = 34108fb22ba95b457bf4b357407b176dcbf3a6db6227227b21ecee045502a16f && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json | cut -d ' ' -f1)\" = 70cd4e923d049da96bcfa4a5b460e2ff5f7460881d5cfd0621607080fd70f68f && test \"$(sha256sum /usr/local/lib/python3.12/dist-packages/vllm/config/vllm.py | cut -d ' ' -f1)\" = 852c2277bd99858cf940238856e7b7f51e9011f768d3c704c930bb9deaf73f61 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json" ] } diff --git a/scripts/config/glm53-flash-e10536a-mtp5-sparkcache-tp4-dcp1.example.json b/scripts/config/glm53-flash-e10536a-mtp5-sparkcache-tp4-dcp1.example.json index 4138edc6..82519fd3 100644 --- a/scripts/config/glm53-flash-e10536a-mtp5-sparkcache-tp4-dcp1.example.json +++ b/scripts/config/glm53-flash-e10536a-mtp5-sparkcache-tp4-dcp1.example.json @@ -81,7 +81,7 @@ "--async-scheduling", "--enable-prefix-caching", "--cudagraph-metrics", - "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"e21f4a0ef4e53c995b7b003e05d8b028abf262b88203ab8b8bc21d6a44462dd6\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_native_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_native_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_native_library_sha256\":\"REPLACE_WITH_NATIVE_LIBRARY_SHA256\",\"spark_cache_native_arena_bytes\":268435456,\"spark_cache_native_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-e10536a-mtp5-static-initialization\"}}" + "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"e21f4a0ef4e53c995b7b003e05d8b028abf262b88203ab8b8bc21d6a44462dd6\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_cuda_placement_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_cuda_placement_library_sha256\":\"REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256\",\"spark_cache_cuda_placement_arena_bytes\":268435456,\"spark_cache_cuda_restore_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-e10536a-mtp5-static-initialization\"}}" ], "extra_volumes": [ {"host": "/REPLACE/GLM53_CACHE_HOST_ROOT", "container": "/cache/jit", "mode": "rw"} @@ -126,6 +126,6 @@ "attestation_hook": [ "/bin/sh", "-ec", - "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && test \"$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location(\"source_identity\", \"/opt/sparkcache-source-identity.py\"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path(\"/opt/sparkcache-src/sparkcache\")))')\" = 34108fb22ba95b457bf4b357407b176dcbf3a6db6227227b21ecee045502a16f && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json | cut -d ' ' -f1)\" = 70cd4e923d049da96bcfa4a5b460e2ff5f7460881d5cfd0621607080fd70f68f && test \"$(sha256sum /usr/local/lib/python3.12/dist-packages/vllm/config/vllm.py | cut -d ' ' -f1)\" = 852c2277bd99858cf940238856e7b7f51e9011f768d3c704c930bb9deaf73f61 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_NATIVE_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json" + "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && test \"$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.spec_from_file_location(\"source_identity\", \"/opt/sparkcache-source-identity.py\"); module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module); print(module.source_tree_sha256(pathlib.Path(\"/opt/sparkcache-src/sparkcache\")))')\" = 34108fb22ba95b457bf4b357407b176dcbf3a6db6227227b21ecee045502a16f && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json | cut -d ' ' -f1)\" = 70cd4e923d049da96bcfa4a5b460e2ff5f7460881d5cfd0621607080fd70f68f && test \"$(sha256sum /usr/local/lib/python3.12/dist-packages/vllm/config/vllm.py | cut -d ' ' -f1)\" = 852c2277bd99858cf940238856e7b7f51e9011f768d3c704c930bb9deaf73f61 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-e10536a.json" ] } diff --git a/scripts/config/glm53-flash-public-python-overlay-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json b/scripts/config/glm53-flash-public-python-overlay-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json new file mode 100644 index 00000000..7ea62557 --- /dev/null +++ b/scripts/config/glm53-flash-public-python-overlay-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json @@ -0,0 +1,163 @@ +{ + "schema": "sparkring-runtime-profile/v1", + "profile_id": "glm53-flash-public-python-overlay-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1", + "model_family": "glm53-flash", + "engine": "docker", + "container_name": "glm53-flash-public-python-overlay-mtp5-adaptive-fastsafetensors-sparkcache-tp4", + "image": "REPLACE_WITH_PUBLIC_PYTHON_OVERLAY_SPARKCACHE_IMAGE", + "image_id": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "model_host_path": "/REPLACE/TARGET_MODEL_HOST_PATH", + "model_container_path": "/models/target", + "shm_size": "32g", + "startup_timeout_seconds": 7200, + "environment": { + "B12X_CUTE_COMPILE_CACHE_DIR": "/cache/jit/b12x/b1d541f9/cute", + "CMAKE_CUDA_ARCHITECTURES": "121", + "CUTE_DSL_ARCH": "sm_121a", + "FLASHINFER_CUDA_ARCH_LIST": "12.1f", + "HF_HUB_OFFLINE": "1", + "INSTANTTENSOR_BACKEND": "AIO_BUFFERED", + "INSTANTTENSOR_BUFFER_SIZE": "1268776960", + "INSTANTTENSOR_CHUNK_SIZE": "2097152", + "INSTANTTENSOR_CONCURRENCY": "1", + "INSTANTTENSOR_IO_DEPTH": "3", + "INSTANTTENSOR_MAX_FREE_MEM_USAGE": "0.05", + "LD_PRELOAD": "/opt/sparkring/nccl/libnccl.so.2", + "NCCL_ALGO": "Ring", + "NCCL_CROSS_NIC": "1", + "NCCL_CUMEM_ENABLE": "0", + "NCCL_DEBUG": "WARN", + "NCCL_IB_DISABLE": "0", + "NCCL_IB_MERGE_NICS": "0", + "NCCL_IB_SUBNET_AWARE_ROUTING": "1", + "NCCL_IGNORE_CPU_AFFINITY": "1", + "NCCL_MAX_NCHANNELS": "4", + "NCCL_MIN_NCHANNELS": "4", + "NCCL_NET": "IB", + "NCCL_NET_PLUGIN": "none", + "NCCL_P2P_LEVEL": "SYS", + "NCCL_PROTO": "LL,LL128,Simple", + "NCCL_SWITCHLESS_RING_ONLY": "1", + "OMP_NUM_THREADS": "16", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "TORCHINDUCTOR_CACHE_DIR": "/cache/jit/torchinductor/vllm-py-0b67266-native-da4d7be", + "TORCHINDUCTOR_COMPILE_THREADS": "1", + "TORCH_CUDA_ARCH_LIST": "12.1a", + "TRANSFORMERS_OFFLINE": "1", + "TRITON_CACHE_DIR": "/cache/jit/triton/vllm-py-0b67266-native-da4d7be", + "VLLM_B12X_MOE_FP4_FORCE_A16": "0", + "VLLM_CACHE_ROOT": "/cache/jit/vllm/py-0b67266-native-da4d7be", + "VLLM_ENABLE_PCIE_ALLREDUCE": "0", + "VLLM_FASTSAFETENSORS_QUEUE_SIZE": "1", + "VLLM_NCCL_SO_PATH": "/opt/sparkring/nccl/libnccl.so.2", + "VLLM_NO_USAGE_STATS": "1", + "VLLM_PLUGINS": "", + "XDG_CACHE_HOME": "/cache/jit" + }, + "extra_vllm_args": [ + "--served-model-name", "glm-5.3-flash-nvfp4-python-overlay-0b67266-on-da4d7be-b12x-b1d541f-mtp5-adaptive-tp4", + "--host", "0.0.0.0", + "--pipeline-parallel-size", "1", + "--disable-custom-all-reduce", + "--mamba-cache-mode", "align", + "--enable-chunked-prefill", + "--dtype", "bfloat16", + "--kv-cache-dtype", "fp8", + "--quantization", "modelopt_mixed", + "--attention-backend", "B12X", + "--block-size", "256", + "--moe-backend", "b12x", + "--linear-backend", "b12x", + "--no-enable-flashinfer-autotune", + "--load-format", "fastsafetensors", + "--enable-auto-tool-choice", + "--tool-call-parser", "glm47", + "--reasoning-parser", "glm45", + "--kda-prefill-backend", "triton", + "--gpu-memory-utilization", "0.80", + "--max-num-batched-tokens", "8192", + "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":5,\"moe_backend\":\"humming\",\"attention_backend\":\"B12X\",\"adaptive_speculative_tokens_initial\":3,\"adaptive_speculative_tokens_window\":32}", + "--compilation-config", "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[8,16,32,64,128,256],\"custom_ops\":[\"all\"]}", + "--max-cudagraph-capture-size", "256", + "--async-scheduling", + "--enable-prefix-caching", + "--cudagraph-metrics", + "--kv-transfer-config", "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context/vllm-py-0b67266-native-da4d7be-b12x-b1d541f-tail-cow\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_publication_schema\":\"tail-cow-v1\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"2e06d909ce5bb71c0c0e3e8be74a70e3b41d92ba4c30196cfb0957fb812acef6\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_cuda_placement_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_cuda_placement_library_sha256\":\"REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256\",\"spark_cache_cuda_placement_arena_bytes\":268435456,\"spark_cache_cuda_restore_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-tail-cow-initialization\"}}" + ], + "extra_volumes": [ + { + "host": "/REPLACE/GLM53_PUBLIC_PYTHON_OVERLAY_CACHE_HOST_ROOT", + "container": "/cache/jit", + "mode": "rw" + } + ], + "extra_labels": { + "org.sparkring.model-profile": "glm53-flash-public-python-overlay-mtp5-adaptive-fastsafetensors-tp4-dcp1", + "org.sparkring.external-cache": "sparkcache", + "org.sparkcache.publication-schema": "tail-cow-v1" + }, + "init": true, + "security_opts": ["label=disable"], + "privileged": false, + "confirmation": "START_GLM53_FLASH_PUBLIC_PYTHON_OVERLAY_MTP5_ADAPTIVE_FASTSAFETENSORS_TP4", + "identity": { + "target_repository": "local-inference-lab/GLM-5.3-Flash-NVFP4", + "target_revision": "520de24eabf507659eaef7c70f14fd584527facc", + "target_cache_identity_sha256": "a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9", + "target_config_sha256": "676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996", + "target_weight_index_sha256": "0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb", + "speculator": "embedded_mtp", + "mtp_maximum_tokens": "5", + "mtp_adaptive": "true", + "mtp_adaptive_initial_tokens": "3", + "mtp_adaptive_window": "32", + "mtp_cache_identity_schema": "glm53-embedded-mtp-composed-runtime-v1", + "mtp_cache_identity_sha256": "2e06d909ce5bb71c0c0e3e8be74a70e3b41d92ba4c30196cfb0957fb812acef6", + "weight_loader": "fastsafetensors", + "weight_loader_queue_size": "1", + "weight_loader_tp_nogds": "true", + "sparkcache_publication_schema": "tail-cow-v1", + "sparkcache_effective_publication_schema": "page-tail-cow-v1", + "sparkcache_source_sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", + "sparkcache_source_revision": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "sparkcache_source_tree": "e864ed9ad64f771188fdb59aa9738e348134d636", + "vllm_native_revision": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "vllm_python_revision": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "vllm_python_tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "b12x_revision": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "b12x_tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "vllm_python_overlay_manifest_sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a" + }, + "required_image_labels": { + "org.jovian.architecture": "linux-arm64-sm121", + "org.jovian.b12x.commit": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "org.jovian.transport": "sparkring-nccl-2.30.7-source-built", + "org.jovian.vllm.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.sparkring.vllm.native.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.sparkring.vllm.python.commit": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "org.sparkring.vllm.python.tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "org.sparkring.vllm.python-overlay-manifest-sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a", + "org.sparkring.vllm.dflash-draft-loader-patch-sha256": "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279", + "org.sparkring.vllm.dflash-draft-loader-postimage-sha256": "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4", + "org.sparkring.vllm.native-elf-manifest-sha256": "REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256", + "org.sparkring.vllm.native-dispatch-manifest-sha256": "REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256", + "org.sparkring.b12x.tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "org.opencontainers.image.base.name": "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd", + "org.sparkring.base.image-id": "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5", + "org.sparkcache.deployment-profile": "glm53-flash-adaptive-mtp-python-overlay", + "org.sparkcache.cuda-placement-library-sha256": "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256", + "org.sparkcache.source-revision": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "org.sparkcache.source-tree": "e864ed9ad64f771188fdb59aa9738e348134d636", + "org.sparkcache.source-sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", + "org.sparkcache.vllm-contract-sha256": "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024", + "org.sparkring.source-receipt-sha256": "REPLACE_WITH_SOURCE_RECEIPT_SHA256", + "org.sparkring.nccl.commit": "73cf112295c33aee2b895f329f592f2a9b4b0f97", + "org.sparkring.nccl.patch-sha256": "6709063fa1c25055ae77a9397dea5d89643f8211d25e7990bdd11597d08c0dde", + "org.sparkring.nccl.patched-tree": "abdeb053b94c3f6d472cd55ae2b79ca821299009" + }, + "attestation_hook": [ + "/bin/sh", + "-ec", + "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && grep -q '\"tail-cow-v1\"' /opt/sparkcache-src/sparkcache/spark_context_cache_config.py && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/vllm-python-overlay.json | cut -d ' ' -f1)\" = e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/source-receipt.json | cut -d ' ' -f1)\" = REPLACE_WITH_SOURCE_RECEIPT_SHA256 && test \"$(python3 -c 'import json; print(json.load(open(\"/opt/sparkring/runtime/python-overlay/retained-native.json\", encoding=\"utf-8\"))[\"native_elf_manifest_sha256\"])')\" = REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256 && test \"$(python3 -c 'import json; print(json.load(open(\"/opt/sparkring/runtime/python-overlay/retained-native.json\", encoding=\"utf-8\"))[\"native_dispatch_manifest_sha256\"])')\" = REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256 && test \"$(cat /opt/sparkring/runtime/python-overlay/sparkcache-source-tree.sha256)\" = f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json | cut -d ' ' -f1)\" = 6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json" + ] +} diff --git a/scripts/prepare_glm53_b12x_kda_adaptive_mtp_profile.py b/scripts/prepare_glm53_b12x_kda_adaptive_mtp_profile.py index a9d8f27c..b9aa99a8 100644 --- a/scripts/prepare_glm53_b12x_kda_adaptive_mtp_profile.py +++ b/scripts/prepare_glm53_b12x_kda_adaptive_mtp_profile.py @@ -11,15 +11,23 @@ import yaml +from sparkcache_terminology import ( + SparkCacheTerminologyError, + canonicalize_profile_connector_arguments, + resolve_string_alias, +) + SHA256 = re.compile(r"[0-9a-f]{64}\Z") SHA256_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") -NATIVE_PLACEHOLDER = "REPLACE_WITH_NATIVE_LIBRARY_SHA256" +CUDA_PLACEMENT_PLACEHOLDER = "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256" +LEGACY_PLACEMENT_PLACEHOLDER = "REPLACE_WITH_NATIVE_LIBRARY_SHA256" IMAGE_PLACEHOLDER = "REPLACE_WITH_B12X_KDA_ADAPTIVE_MTP_SPARKCACHE_IMAGE" PARENT_PLACEHOLDER = "REPLACE_WITH_B12X_KDA_ADAPTIVE_MTP_RUNTIME_IMAGE" -SPARKCACHE_COMMIT = "20838ace3ebda570ca039cb7f1976c29da554b39" +SPARKCACHE_COMMIT = "5d571018de5b63a9a90e5c11e6d6e86bbff4a957" +SPARKCACHE_TREE = "e864ed9ad64f771188fdb59aa9738e348134d636" SPARKCACHE_SOURCE_SHA256 = ( - "4998b24f4f504aeeb9bf92769ec720e282f546e6726d89fdfd06c4efa8d17c10" + "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88" ) LEASE_CONTRACT_SHA256 = ( "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024" @@ -34,13 +42,18 @@ class ResolveError(ValueError): """A source-built image identity or profile template is incomplete.""" -def _replace_native(value: Any, digest: str) -> Any: +def _replace_cuda_placement(value: Any, digest: str) -> Any: if isinstance(value, str): - return value.replace(NATIVE_PLACEHOLDER, digest) + return value.replace(CUDA_PLACEMENT_PLACEHOLDER, digest).replace( + LEGACY_PLACEMENT_PLACEHOLDER, digest + ) if isinstance(value, list): - return [_replace_native(item, digest) for item in value] + return [_replace_cuda_placement(item, digest) for item in value] if isinstance(value, dict): - return {key: _replace_native(item, digest) for key, item in value.items()} + return { + key: _replace_cuda_placement(item, digest) + for key, item in value.items() + } return value @@ -59,8 +72,19 @@ def resolve( image_id: str, parent_image: str, parent_image_id: str, - native_library_sha256: str, + cuda_placement_library_sha256: str | None = None, + native_library_sha256: str | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: + try: + profile = canonicalize_profile_connector_arguments(profile) + cuda_placement_library_sha256 = resolve_string_alias( + cuda_placement_library_sha256, + native_library_sha256, + canonical_name="cuda_placement_library_sha256", + legacy_name="native_library_sha256", + ) + except SparkCacheTerminologyError as error: + raise ResolveError(str(error)) from error if not image or IMAGE_PLACEHOLDER in image: raise ResolveError("SparkCache image reference is unresolved") if not parent_image or PARENT_PLACEHOLDER in parent_image: @@ -69,11 +93,15 @@ def resolve( raise ResolveError("SparkCache image ID must be sha256 plus 64 lowercase hex") if SHA256_ID.fullmatch(parent_image_id) is None: raise ResolveError("runtime parent image ID must be sha256 plus 64 lowercase hex") - if SHA256.fullmatch(native_library_sha256) is None: - raise ResolveError("native library SHA-256 must be 64 lowercase hex") + if SHA256.fullmatch(cuda_placement_library_sha256) is None: + raise ResolveError( + "SparkCache CUDA placement library SHA-256 must be 64 lowercase hex" + ) identity = profile.get("identity", {}) if identity.get("sparkcache_source_revision") != SPARKCACHE_COMMIT: raise ResolveError("profile does not name the integrated SparkCache commit") + if identity.get("sparkcache_source_tree") != SPARKCACHE_TREE: + raise ResolveError("profile does not name the integrated SparkCache Git tree") if identity.get("sparkcache_source_sha256") != SPARKCACHE_SOURCE_SHA256: raise ResolveError("profile does not name the integrated SparkCache source") if identity.get("vllm_revision") != VLLM_COMMIT: @@ -110,7 +138,7 @@ def resolve( if LEASE_CONTRACT_SHA256 not in attestation: raise ResolveError("profile does not attest the live-tensor KDA lease contract") - profile = _replace_native(profile, native_library_sha256) + profile = _replace_cuda_placement(profile, cuda_placement_library_sha256) profile["image"] = image profile["image_id"] = image_id labels = profile["required_image_labels"] @@ -133,7 +161,8 @@ def main() -> int: parser.add_argument("--image-id", required=True) parser.add_argument("--parent-image", required=True) parser.add_argument("--parent-image-id", required=True) - parser.add_argument("--native-library-sha256", required=True) + parser.add_argument("--cuda-placement-library-sha256") + parser.add_argument("--native-library-sha256") parser.add_argument("--profile-output", type=Path, required=True) parser.add_argument("--site-output", type=Path, required=True) args = parser.parse_args() @@ -147,6 +176,9 @@ def main() -> int: image_id=args.image_id, parent_image=args.parent_image, parent_image_id=args.parent_image_id, + cuda_placement_library_sha256=( + args.cuda_placement_library_sha256 + ), native_library_sha256=args.native_library_sha256, ) except (OSError, KeyError, json.JSONDecodeError, ResolveError) as exc: diff --git a/scripts/prepare_glm53_dflash7_python_overlay_profile.py b/scripts/prepare_glm53_dflash7_python_overlay_profile.py new file mode 100644 index 00000000..2c501028 --- /dev/null +++ b/scripts/prepare_glm53_dflash7_python_overlay_profile.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +"""Resolve an exact GLM-5.3 DFlash7 Python-overlay profile and TP4 site.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any + +import yaml + + +SHA256 = re.compile(r"[0-9a-f]{64}\Z") +SHA256_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") +IMAGE_PLACEHOLDER = "REPLACE_WITH_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_IMAGE" +PLACEHOLDERS = { + "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256": "cuda_placement_library_sha256", + "REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256": "native_elf_manifest_sha256", + "REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256": ( + "native_dispatch_manifest_sha256" + ), + "REPLACE_WITH_SOURCE_RECEIPT_SHA256": "source_receipt_sha256", +} +PUBLIC_BASE = ( + "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:" + "864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd" +) +PUBLIC_BASE_ID = "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5" +VLLM_NATIVE_COMMIT = "da4d7be6c97434f6942292ed8abbf4b32dc44355" +VLLM_PYTHON_COMMIT = "0b67266a0f37d6146a8403fb8482403c62f412d5" +VLLM_PYTHON_TREE = "ba9484ccb33aa56e90ff2f447f15ca9b9da97639" +B12X_COMMIT = "b1d541f9e71a35f030d45fae437630fff7507c2a" +B12X_TREE = "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae" +OVERLAY_MANIFEST_SHA256 = ( + "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a" +) +DFLASH_LOADER_PATCH_SHA256 = ( + "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279" +) +DFLASH_LOADER_POSTIMAGE_SHA256 = ( + "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4" +) +SPARKCACHE_COMMIT = "5ec6a9953ad5d39120298bbfc26e95a6fa4b1dc3" +SPARKCACHE_TREE = "94c236b9dfbf5f70075eb47877fd9caaa5d8c249" +SPARKCACHE_SOURCE_SHA256 = ( + "bc238f96e550c7ec27d4081dd1f2e741d404aaf5c8572d89ccc5e76812be4d63" +) +LEASE_CONTRACT_SHA256 = ( + "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024" +) +TARGET_IDENTITY = "a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9" +DFLASH_CONFIG_SHA256 = ( + "c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573" +) +DFLASH_WEIGHTS_SHA256 = ( + "b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b" +) +DEEP_EP_REMOVAL_RECEIPT_SHA256 = ( + "65514f44829e7d176b0b2cacc9559ed22724e525b7041a8bcd4d2e02d1f372e3" +) +DEEP_EP_DISTRIBUTION = "deep_ep==2.0.0+local" +ALLOWED_RUNTIME_WARNINGS = ( + "modelopt_experimental_quantization,fp8_kv_cache_accuracy" +) + + +class ResolveError(ValueError): + """The DFlash7 image identity or runtime profile is incomplete.""" + + +def _argument(profile: dict[str, Any], option: str) -> str: + arguments = profile.get("extra_vllm_args", []) + if arguments.count(option) != 1: + raise ResolveError(f"profile must contain exactly one {option}") + return str(arguments[arguments.index(option) + 1]) + + +def _replace(value: Any, replacements: dict[str, str]) -> Any: + if isinstance(value, str): + for old, new in replacements.items(): + value = value.replace(old, new) + return value + if isinstance(value, list): + return [_replace(item, replacements) for item in value] + if isinstance(value, dict): + return {key: _replace(item, replacements) for key, item in value.items()} + return value + + +def _require_sha256(value: str, label: str) -> None: + if SHA256.fullmatch(value) is None: + raise ResolveError(f"{label} must contain 64 lowercase hexadecimal characters") + + +def resolve( + profile: dict[str, Any], + site: dict[str, Any], + *, + image: str, + image_id: str, + cuda_placement_library_sha256: str, + native_elf_manifest_sha256: str, + native_dispatch_manifest_sha256: str, + source_receipt_sha256: str, +) -> tuple[dict[str, Any], dict[str, Any]]: + if not image or IMAGE_PLACEHOLDER in image: + raise ResolveError("DFlash7 Python-overlay image reference is unresolved") + if SHA256_ID.fullmatch(image_id) is None: + raise ResolveError("DFlash7 image ID must be sha256 plus 64 lowercase hex") + supplied = { + "cuda_placement_library_sha256": cuda_placement_library_sha256, + "native_elf_manifest_sha256": native_elf_manifest_sha256, + "native_dispatch_manifest_sha256": native_dispatch_manifest_sha256, + "source_receipt_sha256": source_receipt_sha256, + } + for name, value in supplied.items(): + _require_sha256(value, name.replace("_", " ")) + + identity = profile.get("identity", {}) + fixed_identity = { + "speculator": "external_dflash", + "draft_weights_sha256": DFLASH_WEIGHTS_SHA256, + "draft_config_sha256": DFLASH_CONFIG_SHA256, + "draft_speculative_tokens": "7", + "draft_tensor_parallel_size": "4", + "kv_cache_dtype": "fp8", + "vllm_block_size": "256", + "max_num_seqs": "32", + "sparkcache_publication_schema": "tail-cow-v1", + "sparkcache_effective_publication_schema": "page-tail-cow-v1", + "sparkcache_source_revision": SPARKCACHE_COMMIT, + "sparkcache_source_tree": SPARKCACHE_TREE, + "sparkcache_source_sha256": SPARKCACHE_SOURCE_SHA256, + "vllm_native_revision": VLLM_NATIVE_COMMIT, + "vllm_python_revision": VLLM_PYTHON_COMMIT, + "vllm_python_tree": VLLM_PYTHON_TREE, + "b12x_revision": B12X_COMMIT, + "b12x_tree": B12X_TREE, + "vllm_python_overlay_manifest_sha256": OVERLAY_MANIFEST_SHA256, + "deep_ep_removed_distribution": DEEP_EP_DISTRIBUTION, + "deep_ep_module_status": "absent", + "allowed_runtime_warnings": ALLOWED_RUNTIME_WARNINGS, + } + for name, expected in fixed_identity.items(): + if identity.get(name) != expected: + raise ResolveError(f"profile identity {name} must be {expected}") + + speculative = json.loads(_argument(profile, "--speculative-config")) + loader = _argument(profile, "--load-format") + if loader != identity.get("target_weight_loader"): + raise ResolveError("target loader argument and profile identity differ") + if loader == "fastsafetensors": + if profile.get("environment", {}).get("VLLM_FASTSAFETENSORS_QUEUE_SIZE") != "1": + raise ResolveError("fastsafetensors queue size must be one") + if identity.get("dflash_peak_gpu_memory_status") != "implemented": + raise ResolveError("separated fastsafetensors DFlash status must be implemented") + if speculative.get("draft_load_config") != {"load_format": "safetensors"}: + raise ResolveError( + "fastsafetensors target requires draft_load_config safetensors" + ) + elif loader == "safetensors": + if identity.get("dflash_peak_gpu_memory_status") != "implemented": + raise ResolveError("safetensors DFlash loader status must be implemented") + else: + raise ResolveError("DFlash7 target loader must be safetensors or fastsafetensors") + + expected_speculative = { + "method": "dflash", + "model": "/dflash-draft", + "num_speculative_tokens": 7, + "draft_tensor_parallel_size": 4, + } + for name, expected in expected_speculative.items(): + if speculative.get(name) != expected: + raise ResolveError(f"DFlash configuration {name} must be {expected}") + + environment = profile.get("environment", {}) + environment_contract = { + "LD_PRELOAD": "/opt/sparkring/nccl/libnccl.so.2", + "VLLM_NCCL_SO_PATH": "/opt/sparkring/nccl/libnccl.so.2", + "NCCL_ALGO": "Ring", + "NCCL_NET": "IB", + "VLLM_ENABLE_PCIE_ALLREDUCE": "0", + "VLLM_ALLREDUCE_USE_FLASHINFER": "0", + "VLLM_ALLREDUCE_USE_SYMM_MEM": "0", + } + for name, expected in environment_contract.items(): + if environment.get(name) != expected: + raise ResolveError(f"runtime environment {name} must be {expected}") + if "OMP_NUM_THREADS" in environment: + raise ResolveError("OMP_NUM_THREADS must remain unset for vLLM thread selection") + if "PYTHONWARNINGS" in environment: + raise ResolveError("runtime warnings must remain visible") + + arguments = profile.get("extra_vllm_args", []) + for flag in ("--disable-custom-all-reduce", "--language-model-only"): + if arguments.count(flag) != 1: + raise ResolveError(f"profile must contain exactly one {flag}") + backend_contract = { + "--attention-backend": "B12X", + "--moe-backend": "b12x", + "--linear-backend": "b12x", + "--kv-cache-dtype": "fp8", + "--quantization": "modelopt_mixed", + } + for option, expected in backend_contract.items(): + if _argument(profile, option) != expected: + raise ResolveError(f"runtime backend {option} must be {expected}") + compilation = json.loads(_argument(profile, "--compilation-config")) + if compilation.get("pass_config", {}).get("fuse_allreduce_rms") is not False: + raise ResolveError("compilation must disable the all-reduce RMS fusion") + + transfer = json.loads(_argument(profile, "--kv-transfer-config")) + extra = transfer.get("kv_connector_extra_config", {}) + cuda_contract = { + "spark_cache_publication_schema": "tail-cow-v1", + "spark_cache_draft_checkpoint_sha256": DFLASH_WEIGHTS_SHA256, + "spark_cache_cuda_restore": True, + "spark_cache_cuda_placement_library": ( + "/opt/sparkcache-src/sparkcache/native/build-cuda/" + "libspark_cache_placement.so" + ), + "spark_cache_cuda_placement_library_sha256": ( + "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256" + ), + "spark_cache_cuda_placement_arena_bytes": 256 * 1024**2, + "spark_cache_cuda_restore_io_workers": 8, + "spark_cache_load_threads": 2, + } + for name, expected in cuda_contract.items(): + if extra.get(name) != expected: + raise ResolveError(f"SparkCache CUDA restore setting {name} must be {expected}") + + labels = profile.get("required_image_labels", {}) + fixed_labels = { + "org.jovian.vllm.commit": VLLM_NATIVE_COMMIT, + "org.sparkring.vllm.native.commit": VLLM_NATIVE_COMMIT, + "org.sparkring.vllm.python.commit": VLLM_PYTHON_COMMIT, + "org.sparkring.vllm.python.tree": VLLM_PYTHON_TREE, + "org.sparkring.vllm.python-overlay-manifest-sha256": OVERLAY_MANIFEST_SHA256, + "org.sparkring.vllm.dflash-draft-loader-patch-sha256": ( + DFLASH_LOADER_PATCH_SHA256 + ), + "org.sparkring.vllm.dflash-draft-loader-postimage-sha256": ( + DFLASH_LOADER_POSTIMAGE_SHA256 + ), + "org.jovian.b12x.commit": B12X_COMMIT, + "org.sparkring.b12x.tree": B12X_TREE, + "org.opencontainers.image.base.name": PUBLIC_BASE, + "org.sparkring.base.image-id": PUBLIC_BASE_ID, + "org.sparkcache.deployment-profile": "glm53-flash-dflash7-python-overlay", + "org.sparkcache.cuda-config-schema": "canonical-v1", + "org.sparkcache.source-revision": SPARKCACHE_COMMIT, + "org.sparkcache.source-tree": SPARKCACHE_TREE, + "org.sparkcache.source-sha256": SPARKCACHE_SOURCE_SHA256, + "org.sparkcache.vllm-contract-sha256": LEASE_CONTRACT_SHA256, + "org.sparkring.runtime.removed-deep-ep-distribution": DEEP_EP_DISTRIBUTION, + "org.sparkring.runtime.deep-ep-removal-receipt-sha256": ( + DEEP_EP_REMOVAL_RECEIPT_SHA256 + ), + } + for name, expected in fixed_labels.items(): + if labels.get(name) != expected: + raise ResolveError(f"profile image label {name} must be {expected}") + + attestation = " ".join(str(item) for item in profile.get("attestation_hook", [])) + for required in ( + DFLASH_CONFIG_SHA256, + DFLASH_WEIGHTS_SHA256, + SPARKCACHE_SOURCE_SHA256, + LEASE_CONTRACT_SHA256, + DEEP_EP_REMOVAL_RECEIPT_SHA256, + 'find_spec("deep_ep") is None', + "/opt/sparkring/runtime/python-overlay/sparkcache-source-tree.sha256", + *PLACEHOLDERS, + ): + if required not in attestation: + raise ResolveError(f"profile attestation omits {required}") + if "adaptive" in " ".join(str(value) for value in labels.values()).lower(): + raise ResolveError("DFlash7 image labels may not claim adaptive MTP") + + replacements = { + placeholder: supplied[name] for placeholder, name in PLACEHOLDERS.items() + } + profile = _replace(profile, replacements) + profile["image"] = image + profile["image_id"] = image_id + site["site"]["name"] = "glm53-flash-dflash7-python-overlay-four-rank-cycle" + site["site"]["description"] = ( + "Four DGX Spark systems serving external DFlash7 through the exact " + "vLLM Python-overlay and SparkCache CUDA restore contracts." + ) + site["runtime"]["container_image"] = image + site["runtime"]["container_image_digest"] = image_id + site["paths"]["jit_cache_dir"] = "/var/lib/sparkring/glm53-dflash7-python-overlay/jit" + site["paths"]["context_cache_dir"] = ( + "/var/lib/sparkring/glm53-dflash7-python-overlay/context" + ) + site["paths"]["evidence_dir"] = "./evidence/glm53-dflash7-python-overlay" + serving = site["serving"] + if serving["tensor_parallel_size"] != 4: + raise ResolveError("DFlash7 site tensor parallel size must be four") + if serving["decode_context_parallel_size"] != 1: + raise ResolveError("DFlash7 site DCP size must be one") + if serving["max_num_seqs"] != 32: + raise ResolveError("DFlash7 site max_num_seqs must be 32") + if "REPLACE_WITH" in json.dumps(profile): + raise ResolveError("resolved DFlash7 profile still contains a placeholder") + return profile, site + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile-template", type=Path, required=True) + parser.add_argument("--site-template", type=Path, required=True) + parser.add_argument("--image", required=True) + parser.add_argument("--image-id", required=True) + parser.add_argument("--cuda-placement-library-sha256", required=True) + parser.add_argument("--native-elf-manifest-sha256", required=True) + parser.add_argument("--native-dispatch-manifest-sha256", required=True) + parser.add_argument("--source-receipt-sha256", required=True) + parser.add_argument("--profile-output", type=Path, required=True) + parser.add_argument("--site-output", type=Path, required=True) + args = parser.parse_args() + try: + profile = json.loads(args.profile_template.read_text(encoding="utf-8")) + site = yaml.safe_load(args.site_template.read_text(encoding="utf-8")) + profile, site = resolve( + profile, + site, + image=args.image, + image_id=args.image_id, + cuda_placement_library_sha256=args.cuda_placement_library_sha256, + native_elf_manifest_sha256=args.native_elf_manifest_sha256, + native_dispatch_manifest_sha256=args.native_dispatch_manifest_sha256, + source_receipt_sha256=args.source_receipt_sha256, + ) + except (OSError, KeyError, json.JSONDecodeError, ResolveError) as exc: + parser.error(str(exc)) + args.profile_output.write_text( + json.dumps(profile, indent=2) + "\n", encoding="utf-8", newline="\n" + ) + args.site_output.write_text( + yaml.safe_dump(site, sort_keys=False), encoding="utf-8", newline="\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prepare_glm53_e105_profile.py b/scripts/prepare_glm53_e105_profile.py index b5cb94af..d8efb654 100644 --- a/scripts/prepare_glm53_e105_profile.py +++ b/scripts/prepare_glm53_e105_profile.py @@ -11,10 +11,17 @@ import yaml +from sparkcache_terminology import ( + SparkCacheTerminologyError, + canonicalize_profile_connector_arguments, + resolve_string_alias, +) + SHA256 = re.compile(r"[0-9a-f]{64}\Z") SHA256_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") -NATIVE_PLACEHOLDER = "REPLACE_WITH_NATIVE_LIBRARY_SHA256" +CUDA_PLACEMENT_PLACEHOLDER = "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256" +LEGACY_PLACEMENT_PLACEHOLDER = "REPLACE_WITH_NATIVE_LIBRARY_SHA256" IMAGE_PLACEHOLDER = "REPLACE_WITH_E10536A_SPARKCACHE_IMAGE" PARENT_PLACEHOLDER = "REPLACE_WITH_E10536A_RUNTIME_IMAGE" SPARKCACHE_COMMIT = "eb3690c1aac2b9e86be8d513799dbb64afa53f25" @@ -30,13 +37,18 @@ class ResolveError(ValueError): """A source-built image identity or profile template is incomplete.""" -def _replace_native(value: Any, digest: str) -> Any: +def _replace_cuda_placement(value: Any, digest: str) -> Any: if isinstance(value, str): - return value.replace(NATIVE_PLACEHOLDER, digest) + return value.replace(CUDA_PLACEMENT_PLACEHOLDER, digest).replace( + LEGACY_PLACEMENT_PLACEHOLDER, digest + ) if isinstance(value, list): - return [_replace_native(item, digest) for item in value] + return [_replace_cuda_placement(item, digest) for item in value] if isinstance(value, dict): - return {key: _replace_native(item, digest) for key, item in value.items()} + return { + key: _replace_cuda_placement(item, digest) + for key, item in value.items() + } return value @@ -48,8 +60,19 @@ def resolve( image_id: str, parent_image: str, parent_image_id: str, - native_library_sha256: str, + cuda_placement_library_sha256: str | None = None, + native_library_sha256: str | None = None, ) -> tuple[dict[str, Any], dict[str, Any]]: + try: + profile = canonicalize_profile_connector_arguments(profile) + cuda_placement_library_sha256 = resolve_string_alias( + cuda_placement_library_sha256, + native_library_sha256, + canonical_name="cuda_placement_library_sha256", + legacy_name="native_library_sha256", + ) + except SparkCacheTerminologyError as error: + raise ResolveError(str(error)) from error if not image or IMAGE_PLACEHOLDER in image: raise ResolveError("SparkCache image reference is unresolved") if not parent_image or PARENT_PLACEHOLDER in parent_image: @@ -58,8 +81,10 @@ def resolve( raise ResolveError("SparkCache image ID must be sha256 plus 64 lowercase hex") if SHA256_ID.fullmatch(parent_image_id) is None: raise ResolveError("runtime parent image ID must be sha256 plus 64 lowercase hex") - if SHA256.fullmatch(native_library_sha256) is None: - raise ResolveError("native library SHA-256 must be 64 lowercase hex") + if SHA256.fullmatch(cuda_placement_library_sha256) is None: + raise ResolveError( + "SparkCache CUDA placement library SHA-256 must be 64 lowercase hex" + ) identity = profile.get("identity", {}) if identity.get("sparkcache_source_revision") != SPARKCACHE_COMMIT: raise ResolveError("profile does not name the integrated SparkCache commit") @@ -71,7 +96,7 @@ def resolve( if LEASE_CONTRACT_SHA256 not in attestation: raise ResolveError("profile does not attest the e10536a lease contract") - profile = _replace_native(profile, native_library_sha256) + profile = _replace_cuda_placement(profile, cuda_placement_library_sha256) profile["image"] = image profile["image_id"] = image_id labels = profile["required_image_labels"] @@ -94,7 +119,8 @@ def main() -> int: parser.add_argument("--image-id", required=True) parser.add_argument("--parent-image", required=True) parser.add_argument("--parent-image-id", required=True) - parser.add_argument("--native-library-sha256", required=True) + parser.add_argument("--cuda-placement-library-sha256") + parser.add_argument("--native-library-sha256") parser.add_argument("--profile-output", type=Path, required=True) parser.add_argument("--site-output", type=Path, required=True) args = parser.parse_args() @@ -108,6 +134,9 @@ def main() -> int: image_id=args.image_id, parent_image=args.parent_image, parent_image_id=args.parent_image_id, + cuda_placement_library_sha256=( + args.cuda_placement_library_sha256 + ), native_library_sha256=args.native_library_sha256, ) except (OSError, KeyError, json.JSONDecodeError, ResolveError) as exc: diff --git a/scripts/prepare_glm53_public_python_overlay_profile.py b/scripts/prepare_glm53_public_python_overlay_profile.py new file mode 100644 index 00000000..74c76114 --- /dev/null +++ b/scripts/prepare_glm53_public_python_overlay_profile.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""Resolve the GLM-5.3 public-base Python-overlay profile and TP4 site.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path +from typing import Any + +import yaml + +from sparkcache_terminology import ( + SparkCacheTerminologyError, + canonicalize_profile_connector_arguments, + resolve_string_alias, +) + + +SHA256 = re.compile(r"[0-9a-f]{64}\Z") +SHA256_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") +IMAGE_PLACEHOLDER = "REPLACE_WITH_PUBLIC_PYTHON_OVERLAY_SPARKCACHE_IMAGE" +CUDA_PLACEMENT_PLACEHOLDER = "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256" +LEGACY_PLACEMENT_PLACEHOLDER = "REPLACE_WITH_NATIVE_LIBRARY_SHA256" +CUDA_PLACEMENT_LABEL = "org.sparkcache.cuda-placement-library-sha256" +LEGACY_PLACEMENT_LABEL = "org.sparkcache.native-library-sha256" +PLACEHOLDERS = { + CUDA_PLACEMENT_PLACEHOLDER: "cuda_placement_library_sha256", + "REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256": "native_elf_manifest_sha256", + "REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256": ( + "native_dispatch_manifest_sha256" + ), + "REPLACE_WITH_SOURCE_RECEIPT_SHA256": "source_receipt_sha256", +} +PUBLIC_BASE = ( + "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:" + "864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd" +) +PUBLIC_BASE_ID = "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5" +VLLM_NATIVE_COMMIT = "da4d7be6c97434f6942292ed8abbf4b32dc44355" +VLLM_PYTHON_COMMIT = "0b67266a0f37d6146a8403fb8482403c62f412d5" +VLLM_PYTHON_TREE = "ba9484ccb33aa56e90ff2f447f15ca9b9da97639" +B12X_COMMIT = "b1d541f9e71a35f030d45fae437630fff7507c2a" +B12X_TREE = "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae" +OVERLAY_MANIFEST_SHA256 = ( + "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a" +) +DFLASH_LOADER_PATCH_SHA256 = ( + "39b567013ee7aed79f63200ed460129587933dc77fb430decdf19f78178de279" +) +DFLASH_LOADER_POSTIMAGE_SHA256 = ( + "98acbae2b3bb4482d83f9637c163ce7c92707ccdf6561b7e431f23337f151cf4" +) +SPARKCACHE_COMMIT = "5d571018de5b63a9a90e5c11e6d6e86bbff4a957" +SPARKCACHE_TREE = "e864ed9ad64f771188fdb59aa9738e348134d636" +SPARKCACHE_SOURCE_SHA256 = ( + "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88" +) +LEASE_CONTRACT_SHA256 = ( + "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024" +) +TARGET_IDENTITY = "a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9" +MTP_CACHE_IDENTITY_SHA256 = ( + "2e06d909ce5bb71c0c0e3e8be74a70e3b41d92ba4c30196cfb0957fb812acef6" +) + + +class ResolveError(ValueError): + """The composed runtime identity or profile template is incomplete.""" + + +def composed_mtp_identity() -> str: + fields = ( + "glm53-embedded-mtp-composed-runtime-v1", + TARGET_IDENTITY, + VLLM_PYTHON_COMMIT, + VLLM_NATIVE_COMMIT, + B12X_COMMIT, + "5", + "adaptive:3:32", + ) + return hashlib.sha256("\0".join(fields).encode()).hexdigest() + + +def _replace(value: Any, replacements: dict[str, str]) -> Any: + if isinstance(value, str): + for placeholder, replacement in replacements.items(): + value = value.replace(placeholder, replacement) + return value + if isinstance(value, list): + return [_replace(item, replacements) for item in value] + if isinstance(value, dict): + return {key: _replace(item, replacements) for key, item in value.items()} + return value + + +def _argument(profile: dict[str, Any], option: str) -> str: + arguments = profile.get("extra_vllm_args", []) + if arguments.count(option) != 1: + raise ResolveError(f"profile must contain exactly one {option}") + return str(arguments[arguments.index(option) + 1]) + + +def _require_sha256(value: str, label: str) -> None: + if SHA256.fullmatch(value) is None: + raise ResolveError(f"{label} must contain 64 lowercase hexadecimal characters") + + +def resolve( + profile: dict[str, Any], + site: dict[str, Any], + *, + image: str, + image_id: str, + cuda_placement_library_sha256: str | None = None, + native_elf_manifest_sha256: str, + native_dispatch_manifest_sha256: str, + source_receipt_sha256: str, + native_library_sha256: str | None = None, +) -> tuple[dict[str, Any], dict[str, Any]]: + try: + profile = canonicalize_profile_connector_arguments(profile) + cuda_placement_library_sha256 = resolve_string_alias( + cuda_placement_library_sha256, + native_library_sha256, + canonical_name="cuda_placement_library_sha256", + legacy_name="native_library_sha256", + ) + except SparkCacheTerminologyError as error: + raise ResolveError(str(error)) from error + profile = _replace( + profile, + {LEGACY_PLACEMENT_PLACEHOLDER: CUDA_PLACEMENT_PLACEHOLDER}, + ) + labels = dict(profile.get("required_image_labels", {})) + if ( + CUDA_PLACEMENT_LABEL in labels + and LEGACY_PLACEMENT_LABEL in labels + and labels[CUDA_PLACEMENT_LABEL] != labels[LEGACY_PLACEMENT_LABEL] + ): + raise ResolveError( + f"profile image labels {CUDA_PLACEMENT_LABEL} and compatibility " + f"alias {LEGACY_PLACEMENT_LABEL} have conflicting values" + ) + if LEGACY_PLACEMENT_LABEL in labels and CUDA_PLACEMENT_LABEL not in labels: + labels[CUDA_PLACEMENT_LABEL] = labels[LEGACY_PLACEMENT_LABEL] + labels.pop(LEGACY_PLACEMENT_LABEL, None) + profile["required_image_labels"] = labels + if not image or IMAGE_PLACEHOLDER in image: + raise ResolveError("Python-overlay image reference is unresolved") + if SHA256_ID.fullmatch(image_id) is None: + raise ResolveError("Python-overlay image ID must be sha256 plus 64 lowercase hex") + supplied = { + "cuda_placement_library_sha256": cuda_placement_library_sha256, + "native_elf_manifest_sha256": native_elf_manifest_sha256, + "native_dispatch_manifest_sha256": native_dispatch_manifest_sha256, + "source_receipt_sha256": source_receipt_sha256, + } + for name, value in supplied.items(): + _require_sha256(value, name.replace("_", " ")) + + identity = profile.get("identity", {}) + expected_identity = { + "vllm_native_revision": VLLM_NATIVE_COMMIT, + "vllm_python_revision": VLLM_PYTHON_COMMIT, + "vllm_python_tree": VLLM_PYTHON_TREE, + "b12x_revision": B12X_COMMIT, + "b12x_tree": B12X_TREE, + "vllm_python_overlay_manifest_sha256": OVERLAY_MANIFEST_SHA256, + "sparkcache_source_revision": SPARKCACHE_COMMIT, + "sparkcache_source_tree": SPARKCACHE_TREE, + "sparkcache_source_sha256": SPARKCACHE_SOURCE_SHA256, + "mtp_cache_identity_schema": "glm53-embedded-mtp-composed-runtime-v1", + "mtp_cache_identity_sha256": MTP_CACHE_IDENTITY_SHA256, + "mtp_maximum_tokens": "5", + "mtp_adaptive": "true", + "mtp_adaptive_initial_tokens": "3", + "mtp_adaptive_window": "32", + "weight_loader": "fastsafetensors", + "weight_loader_queue_size": "1", + "weight_loader_tp_nogds": "true", + "sparkcache_publication_schema": "tail-cow-v1", + "sparkcache_effective_publication_schema": "page-tail-cow-v1", + } + for name, expected in expected_identity.items(): + if identity.get(name) != expected: + raise ResolveError(f"profile identity {name} must be {expected}") + if composed_mtp_identity() != MTP_CACHE_IDENTITY_SHA256: + raise ResolveError("composed MTP identity constant is inconsistent") + if _argument(profile, "--load-format") != "fastsafetensors": + raise ResolveError("profile must select the fastsafetensors loader") + speculative = json.loads(_argument(profile, "--speculative-config")) + expected_speculative = { + "num_speculative_tokens": 5, + "adaptive_speculative_tokens_initial": 3, + "adaptive_speculative_tokens_window": 32, + } + for name, expected in expected_speculative.items(): + if speculative.get(name) != expected: + raise ResolveError(f"speculative configuration {name} must be {expected}") + if profile.get("environment", {}).get("VLLM_FASTSAFETENSORS_QUEUE_SIZE") != "1": + raise ResolveError("fastsafetensors queue size must be one") + transfer = json.loads(_argument(profile, "--kv-transfer-config")) + transfer_extra = transfer.get("kv_connector_extra_config", {}) + if transfer_extra.get("spark_cache_publication_schema") != "tail-cow-v1": + raise ResolveError("opaque page-tail publication requires tail-cow-v1") + + labels = profile.get("required_image_labels", {}) + fixed_labels = { + "org.jovian.vllm.commit": VLLM_NATIVE_COMMIT, + "org.sparkring.vllm.native.commit": VLLM_NATIVE_COMMIT, + "org.sparkring.vllm.python.commit": VLLM_PYTHON_COMMIT, + "org.sparkring.vllm.python.tree": VLLM_PYTHON_TREE, + "org.sparkring.vllm.python-overlay-manifest-sha256": OVERLAY_MANIFEST_SHA256, + "org.sparkring.vllm.dflash-draft-loader-patch-sha256": ( + DFLASH_LOADER_PATCH_SHA256 + ), + "org.sparkring.vllm.dflash-draft-loader-postimage-sha256": ( + DFLASH_LOADER_POSTIMAGE_SHA256 + ), + "org.jovian.b12x.commit": B12X_COMMIT, + "org.sparkring.b12x.tree": B12X_TREE, + "org.opencontainers.image.base.name": PUBLIC_BASE, + "org.sparkring.base.image-id": PUBLIC_BASE_ID, + "org.sparkcache.source-revision": SPARKCACHE_COMMIT, + "org.sparkcache.source-tree": SPARKCACHE_TREE, + "org.sparkcache.source-sha256": SPARKCACHE_SOURCE_SHA256, + "org.sparkcache.vllm-contract-sha256": LEASE_CONTRACT_SHA256, + } + for name, expected in fixed_labels.items(): + if labels.get(name) != expected: + raise ResolveError(f"profile image label {name} must be {expected}") + + attestation = " ".join(str(item) for item in profile.get("attestation_hook", [])) + required_attestations = ( + OVERLAY_MANIFEST_SHA256, + SPARKCACHE_SOURCE_SHA256, + LEASE_CONTRACT_SHA256, + '"tail-cow-v1"', + "/opt/sparkring/runtime/python-overlay/sparkcache-source-tree.sha256", + *PLACEHOLDERS, + ) + for required in required_attestations: + if required not in attestation: + raise ResolveError(f"profile attestation omits {required}") + if "source_tree_sha256(" in attestation: + raise ResolveError( + "profile attestation must use the clean SparkCache source receipt" + ) + + replacements = { + placeholder: supplied[name] for placeholder, name in PLACEHOLDERS.items() + } + profile = _replace(profile, replacements) + profile["image"] = image + profile["image_id"] = image_id + site["site"]["name"] = "glm53-flash-public-python-overlay-four-rank-cycle" + site["site"]["description"] = ( + "Four DGX Spark systems serving vLLM 0b67266 Python over retained " + "da4d7be native extensions with B12X b1d541f and SparkCache tail publication." + ) + site["runtime"]["container_image"] = image + site["runtime"]["container_image_digest"] = image_id + site["paths"]["jit_cache_dir"] = ( + "/var/lib/sparkring/glm53-public-python-overlay/jit" + ) + site["paths"]["context_cache_dir"] = ( + "/var/lib/sparkring/glm53-public-python-overlay/context" + ) + site["paths"]["evidence_dir"] = ( + "./evidence/glm53-public-python-overlay" + ) + if site["serving"]["kv_cache_bytes_per_rank"] != 20 * 1024**3: + raise ResolveError("GLM-5.3 site must reserve 20 GiB of FP8 KV per rank") + if "REPLACE_WITH" in json.dumps(profile): + raise ResolveError("resolved profile still contains an artifact placeholder") + return profile, site + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile-template", type=Path, required=True) + parser.add_argument("--site-template", type=Path, required=True) + parser.add_argument("--image", required=True) + parser.add_argument("--image-id", required=True) + parser.add_argument("--cuda-placement-library-sha256") + parser.add_argument("--native-library-sha256") + parser.add_argument("--native-elf-manifest-sha256", required=True) + parser.add_argument("--native-dispatch-manifest-sha256", required=True) + parser.add_argument("--source-receipt-sha256", required=True) + parser.add_argument("--profile-output", type=Path, required=True) + parser.add_argument("--site-output", type=Path, required=True) + args = parser.parse_args() + try: + profile = json.loads(args.profile_template.read_text(encoding="utf-8")) + site = yaml.safe_load(args.site_template.read_text(encoding="utf-8")) + profile, site = resolve( + profile, + site, + image=args.image, + image_id=args.image_id, + cuda_placement_library_sha256=( + args.cuda_placement_library_sha256 + ), + native_elf_manifest_sha256=args.native_elf_manifest_sha256, + native_dispatch_manifest_sha256=args.native_dispatch_manifest_sha256, + source_receipt_sha256=args.source_receipt_sha256, + native_library_sha256=args.native_library_sha256, + ) + except (OSError, KeyError, json.JSONDecodeError, ResolveError) as exc: + parser.error(str(exc)) + args.profile_output.write_text( + json.dumps(profile, indent=2) + "\n", encoding="utf-8", newline="\n" + ) + args.site_output.write_text( + yaml.safe_dump(site, sort_keys=False), encoding="utf-8", newline="\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sparkcache_terminology.py b/scripts/sparkcache_terminology.py new file mode 100644 index 00000000..ff7427b3 --- /dev/null +++ b/scripts/sparkcache_terminology.py @@ -0,0 +1,151 @@ +"""Canonical SparkCache CUDA configuration names and compatibility aliases.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from typing import Any + + +CANONICAL_CONNECTOR_KEYS = { + "spark_cache_cuda_restore": "spark_cache_native_restore", + "spark_cache_cuda_placement_library": "spark_cache_native_library", + "spark_cache_cuda_placement_library_sha256": ( + "spark_cache_native_library_sha256" + ), + "spark_cache_cuda_placement_arena_bytes": "spark_cache_native_arena_bytes", + "spark_cache_cuda_restore_io_workers": "spark_cache_native_io_workers", +} + + +class SparkCacheTerminologyError(ValueError): + """Canonical and compatibility names cannot produce one exact contract.""" + + +def _same_json_value(left: Any, right: Any) -> bool: + return type(left) is type(right) and left == right + + +def canonicalize_connector_extra_config(extra: Mapping[str, Any]) -> dict[str, Any]: + """Return canonical connector keys and reject contradictory aliases.""" + + result = dict(extra) + for canonical, legacy in CANONICAL_CONNECTOR_KEYS.items(): + canonical_present = canonical in result + legacy_present = legacy in result + if canonical_present and legacy_present and not _same_json_value( + result[canonical], result[legacy] + ): + raise SparkCacheTerminologyError( + f"SparkCache connector keys {canonical} and compatibility alias " + f"{legacy} have conflicting values" + ) + if legacy_present and not canonical_present: + result[canonical] = result[legacy] + result.pop(legacy, None) + return result + + +def canonicalize_connector_arguments(arguments: Sequence[str]) -> tuple[str, ...]: + """Normalize SparkCache keys inside one vLLM KV-transfer argument.""" + + result = tuple(arguments) + locations: list[tuple[int, str, str]] = [] + for index, argument in enumerate(result): + if argument == "--kv-transfer-config": + if index + 1 >= len(result): + raise SparkCacheTerminologyError( + "--kv-transfer-config requires a JSON value" + ) + locations.append((index + 1, result[index + 1], "separate")) + elif argument.startswith("--kv-transfer-config="): + locations.append( + ( + index, + argument.split("=", 1)[1], + "equals", + ) + ) + if not locations: + return result + if len(locations) != 1: + raise SparkCacheTerminologyError( + "a runtime profile must contain at most one --kv-transfer-config" + ) + + index, encoded, form = locations[0] + try: + transfer = json.loads(encoded) + except json.JSONDecodeError as error: + raise SparkCacheTerminologyError( + "--kv-transfer-config must contain valid JSON" + ) from error + if not isinstance(transfer, dict): + raise SparkCacheTerminologyError( + "--kv-transfer-config must contain a JSON object" + ) + extra = transfer.get("kv_connector_extra_config") + if extra is None: + return result + if not isinstance(extra, dict): + raise SparkCacheTerminologyError( + "kv_connector_extra_config must contain a JSON object" + ) + canonical = canonicalize_connector_extra_config(extra) + if canonical == extra: + return result + + transfer["kv_connector_extra_config"] = canonical + replacement = json.dumps(transfer, separators=(",", ":")) + mutable = list(result) + mutable[index] = ( + replacement + if form == "separate" + else f"--kv-transfer-config={replacement}" + ) + return tuple(mutable) + + +def canonicalize_profile_connector_arguments( + profile: Mapping[str, Any], +) -> dict[str, Any]: + """Return a profile whose emitted vLLM arguments use canonical keys.""" + + result = dict(profile) + arguments = result.get("extra_vllm_args") + if arguments is None: + return result + if not isinstance(arguments, list) or not all( + isinstance(argument, str) for argument in arguments + ): + raise SparkCacheTerminologyError( + "profile extra_vllm_args must contain a JSON string array" + ) + normalized = canonicalize_connector_arguments(arguments) + if normalized != tuple(arguments): + result["extra_vllm_args"] = list(normalized) + return result + + +def resolve_string_alias( + canonical_value: str | None, + legacy_value: str | None, + *, + canonical_name: str, + legacy_name: str, +) -> str: + """Resolve one CLI/Python alias pair without accepting ambiguity.""" + + if ( + canonical_value is not None + and legacy_value is not None + and canonical_value != legacy_value + ): + raise SparkCacheTerminologyError( + f"{canonical_name} and compatibility alias {legacy_name} " + "have conflicting values" + ) + selected = canonical_value if canonical_value is not None else legacy_value + if selected is None: + raise SparkCacheTerminologyError(f"{canonical_name} is required") + return selected diff --git a/scripts/sparkring_runtime.py b/scripts/sparkring_runtime.py index 3a854999..21bb865d 100644 --- a/scripts/sparkring_runtime.py +++ b/scripts/sparkring_runtime.py @@ -34,6 +34,11 @@ from pathlib import Path from typing import Any, Mapping +from sparkcache_terminology import ( + SparkCacheTerminologyError, + canonicalize_connector_arguments, +) + # --------------------------------------------------------------------------- # Schema constants @@ -514,6 +519,10 @@ def parse_runtime_profile( extra_vllm_args = _validate_extra_vllm_args( document["extra_vllm_args"], f"{where}.extra_vllm_args", ) + try: + extra_vllm_args = canonicalize_connector_arguments(extra_vllm_args) + except SparkCacheTerminologyError as error: + raise ProfileError(f"{where}.extra_vllm_args: {error}") from error extra_volumes = _validate_extra_volumes( document.get("extra_volumes"), f"{where}.extra_volumes", ) diff --git a/scripts/test_glm53_flash_profile.py b/scripts/test_glm53_flash_profile.py index ea6faec4..c4edb2be 100644 --- a/scripts/test_glm53_flash_profile.py +++ b/scripts/test_glm53_flash_profile.py @@ -253,7 +253,8 @@ def test_cache_profile_adds_only_the_external_connector_to_serving_arguments() - assert extra["spark_cache_model_profile"] == "glm53-flash-hybrid" assert extra["spark_cache_draft_policy"] == "separate" assert extra["spark_cache_streaming_snapshots"] is False - assert extra["spark_cache_native_restore"] is False + assert extra["spark_cache_cuda_restore"] is False + assert not any(key.startswith("spark_cache_native_") for key in extra) for attribute in ( "image", diff --git a/scripts/test_prepare_glm53_b12x_kda_adaptive_mtp_profile.py b/scripts/test_prepare_glm53_b12x_kda_adaptive_mtp_profile.py index 8596aea5..d1386c27 100644 --- a/scripts/test_prepare_glm53_b12x_kda_adaptive_mtp_profile.py +++ b/scripts/test_prepare_glm53_b12x_kda_adaptive_mtp_profile.py @@ -14,6 +14,7 @@ MTP_CACHE_IDENTITY_SHA256, SPARKCACHE_COMMIT, SPARKCACHE_SOURCE_SHA256, + SPARKCACHE_TREE, VLLM_COMMIT, ResolveError, resolve, @@ -66,6 +67,7 @@ def test_profile_pins_adaptive_mtp_fastsafetensors_and_sparkcache() -> None: identity = profile["identity"] assert identity["vllm_revision"] == VLLM_COMMIT assert identity["sparkcache_source_revision"] == SPARKCACHE_COMMIT + assert identity["sparkcache_source_tree"] == SPARKCACHE_TREE assert identity["sparkcache_source_sha256"] == SPARKCACHE_SOURCE_SHA256 assert identity["mtp_cache_identity_sha256"] == _mtp_identity() assert identity["mtp_cache_identity_sha256"] == MTP_CACHE_IDENTITY_SHA256 @@ -73,9 +75,10 @@ def test_profile_pins_adaptive_mtp_fastsafetensors_and_sparkcache() -> None: transfer = json.loads(_argument(profile, "--kv-transfer-config")) extra = transfer["kv_connector_extra_config"] assert extra["spark_cache_draft_checkpoint_sha256"] == MTP_CACHE_IDENTITY_SHA256 - assert extra["spark_cache_native_restore"] is True - assert extra["spark_cache_native_arena_bytes"] == 256 * 1024**2 - assert extra["spark_cache_native_io_workers"] == 8 + assert extra["spark_cache_cuda_restore"] is True + assert extra["spark_cache_cuda_placement_arena_bytes"] == 256 * 1024**2 + assert extra["spark_cache_cuda_restore_io_workers"] == 8 + assert not any(key.startswith("spark_cache_native_") for key in extra) assert extra["spark_cache_load_threads"] == 2 assert extra["spark_cache_clear_once"] == ( "sparkring-b12x-kda-adaptive-mtp-fastsafetensors-initialization" @@ -117,7 +120,7 @@ def test_resolver_produces_an_aligned_tp4_profile(tmp_path: Path) -> None: image_id="sha256:" + "b" * 64, parent_image="local/glm53-kda-runtime@sha256:" + "c" * 64, parent_image_id="sha256:" + "d" * 64, - native_library_sha256="e" * 64, + cuda_placement_library_sha256="e" * 64, ) assert site["topology"] and len(site["ranks"]) == 4 assert site["serving"]["tensor_parallel_size"] == 4 @@ -142,19 +145,87 @@ def test_resolver_rejects_runtime_or_loader_identity_drift() -> None: "image_id": "sha256:" + "a" * 64, "parent_image": "parent", "parent_image_id": "sha256:" + "b" * 64, - "native_library_sha256": "c" * 64, + "cuda_placement_library_sha256": "c" * 64, } changed = copy.deepcopy(profile) changed["identity"]["vllm_revision"] = "0" * 40 with pytest.raises(ResolveError, match="live-tensor B12X KDA runtime"): resolve(changed, copy.deepcopy(site), **arguments) + changed = copy.deepcopy(profile) + changed["identity"]["sparkcache_source_tree"] = "0" * 40 + with pytest.raises(ResolveError, match="SparkCache Git tree"): + resolve(changed, copy.deepcopy(site), **arguments) + changed = copy.deepcopy(profile) changed["environment"]["VLLM_FASTSAFETENSORS_QUEUE_SIZE"] = "2" with pytest.raises(ResolveError, match="queue size must be one"): resolve(changed, copy.deepcopy(site), **arguments) +def test_resolver_normalizes_legacy_cuda_restore_aliases_and_rejects_conflicts() -> None: + profile = json.loads(PROFILE.read_text(encoding="utf-8")) + site = yaml.safe_load(SITE.read_text(encoding="utf-8")) + arguments = profile["extra_vllm_args"] + index = arguments.index("--kv-transfer-config") + 1 + transfer = json.loads(arguments[index]) + extra = transfer["kv_connector_extra_config"] + extra["spark_cache_native_restore"] = extra.pop("spark_cache_cuda_restore") + extra["spark_cache_native_library"] = extra.pop( + "spark_cache_cuda_placement_library" + ) + arguments[index] = json.dumps(transfer, separators=(",", ":")) + + resolved, _ = resolve( + profile, + site, + image="image", + image_id="sha256:" + "a" * 64, + parent_image="parent", + parent_image_id="sha256:" + "b" * 64, + native_library_sha256="c" * 64, + ) + resolved_extra = json.loads(_argument(resolved, "--kv-transfer-config"))[ + "kv_connector_extra_config" + ] + assert resolved_extra["spark_cache_cuda_restore"] is True + assert "spark_cache_native_restore" not in resolved_extra + assert "spark_cache_native_library" not in resolved_extra + + conflict = copy.deepcopy(profile) + conflict_arguments = conflict["extra_vllm_args"] + conflict_index = conflict_arguments.index("--kv-transfer-config") + 1 + conflict_transfer = json.loads(conflict_arguments[conflict_index]) + conflict_transfer["kv_connector_extra_config"][ + "spark_cache_cuda_restore" + ] = False + conflict_arguments[conflict_index] = json.dumps( + conflict_transfer, separators=(",", ":") + ) + with pytest.raises(ResolveError, match="conflicting values"): + resolve( + conflict, + copy.deepcopy(site), + image="image", + image_id="sha256:" + "a" * 64, + parent_image="parent", + parent_image_id="sha256:" + "b" * 64, + cuda_placement_library_sha256="c" * 64, + ) + + with pytest.raises(ResolveError, match="conflicting values"): + resolve( + json.loads(PROFILE.read_text(encoding="utf-8")), + copy.deepcopy(site), + image="image", + image_id="sha256:" + "a" * 64, + parent_image="parent", + parent_image_id="sha256:" + "b" * 64, + cuda_placement_library_sha256="c" * 64, + native_library_sha256="d" * 64, + ) + + def test_quickstart_names_the_executable_builder_and_profile_contracts() -> None: guide = QUICKSTART.read_text(encoding="utf-8") assert "runtime/glm53-flash-b12x-kda-adaptive-mtp/build-image.sh" in guide @@ -162,6 +233,9 @@ def test_quickstart_names_the_executable_builder_and_profile_contracts() -> None assert str(SITE.relative_to(ROOT)).replace("\\", "/") in guide assert "prepare_glm53_b12x_kda_adaptive_mtp_profile.py" in guide assert SPARKCACHE_COMMIT in guide + assert SPARKCACHE_TREE in guide assert SPARKCACHE_SOURCE_SHA256 in guide assert VLLM_COMMIT in guide assert "START_GLM53_FLASH_MTP5_ADAPTIVE_FASTSAFETENSORS_TP4" in guide + assert "--cuda-placement-library-sha256" in guide + assert "--native-library-sha256" not in guide diff --git a/scripts/test_prepare_glm53_dflash7_python_overlay_profile.py b/scripts/test_prepare_glm53_dflash7_python_overlay_profile.py new file mode 100644 index 00000000..c8884c90 --- /dev/null +++ b/scripts/test_prepare_glm53_dflash7_python_overlay_profile.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest +import yaml + +import sparkring_generic_launcher as launcher +from prepare_glm53_dflash7_python_overlay_profile import ( + ALLOWED_RUNTIME_WARNINGS, + B12X_COMMIT, + DEEP_EP_DISTRIBUTION, + DEEP_EP_REMOVAL_RECEIPT_SHA256, + DFLASH_LOADER_PATCH_SHA256, + DFLASH_LOADER_POSTIMAGE_SHA256, + DFLASH_WEIGHTS_SHA256, + SPARKCACHE_COMMIT, + SPARKCACHE_SOURCE_SHA256, + SPARKCACHE_TREE, + VLLM_NATIVE_COMMIT, + VLLM_PYTHON_COMMIT, + ResolveError, + resolve, +) + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ROOT / "scripts/config" +FAST = ( + CONFIG + / "glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json" +) +SAFE = ( + CONFIG + / "glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json" +) +SITE = CONFIG / "glm53-flash-tp4-site.example.yaml" +GUIDE = ROOT / "docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md" +DIGESTS = { + "cuda_placement_library_sha256": "1a" * 32, + "native_elf_manifest_sha256": "2b" * 32, + "native_dispatch_manifest_sha256": "3c" * 32, + "source_receipt_sha256": "4d" * 32, +} + + +def _argument(profile: dict, option: str) -> str: + args = profile["extra_vllm_args"] + return args[args.index(option) + 1] + + +def _resolved(path: Path) -> tuple[dict, dict]: + profile = json.loads(path.read_text(encoding="utf-8")) + site = yaml.safe_load(SITE.read_text(encoding="utf-8")) + profile["model_host_path"] = "/srv/models/glm53" + profile["extra_volumes"][0]["host"] = "/srv/models/dflash2" + profile["extra_volumes"][1]["host"] = "/srv/cache/glm53-dflash7" + management = ["10.20.0.10", "10.20.0.11", "10.20.0.12", "10.20.0.13"] + networks = ["10.30.1.0/24", "10.30.2.0/24", "10.30.3.0/24", "10.30.4.0/24"] + ring = ( + ("10.30.1.10", "10.30.4.10"), + ("10.30.1.11", "10.30.2.11"), + ("10.30.2.12", "10.30.3.12"), + ("10.30.3.13", "10.30.4.13"), + ) + for edge, subnet in zip(site["topology"]["edges"], networks, strict=True): + edge["subnet"] = subnet + for rank, address, ports in zip(site["ranks"], management, ring, strict=True): + rank["ssh_target"] = f"operator@{address}" + rank["management"]["address"] = address + for port, port_address in zip(rank["ring_ports"], ports, strict=True): + port["address"] = port_address + for peer in rank["transport_peers"]: + peer["address"] = management[peer["rank"]] + return resolve( + profile, + site, + image="local/glm53-dflash7-overlay@sha256:" + "ab" * 32, + image_id="sha256:" + "cd" * 32, + **DIGESTS, + ) + + +@pytest.mark.parametrize("path,loader", [(FAST, "fastsafetensors"), (SAFE, "safetensors")]) +def test_profiles_pin_external_dflash7_and_tp4(path: Path, loader: str) -> None: + profile = json.loads(path.read_text(encoding="utf-8")) + speculative = json.loads(_argument(profile, "--speculative-config")) + assert speculative["method"] == "dflash" + assert speculative["num_speculative_tokens"] == 7 + assert speculative["draft_tensor_parallel_size"] == 4 + assert _argument(profile, "--load-format") == loader + assert profile["identity"]["draft_weights_sha256"] == DFLASH_WEIGHTS_SHA256 + assert profile["identity"]["max_num_seqs"] == "32" + assert profile["identity"]["vllm_block_size"] == "256" + assert profile["identity"]["kv_cache_dtype"] == "fp8" + assert not any( + name.startswith("INSTANTTENSOR_") for name in profile["environment"] + ) + environment = profile["environment"] + assert "OMP_NUM_THREADS" not in environment + assert "PYTHONWARNINGS" not in environment + assert environment["VLLM_ALLREDUCE_USE_SYMM_MEM"] == "0" + assert environment["VLLM_ALLREDUCE_USE_FLASHINFER"] == "0" + assert environment["VLLM_NCCL_SO_PATH"] == "/opt/sparkring/nccl/libnccl.so.2" + assert environment["LD_PRELOAD"] == environment["VLLM_NCCL_SO_PATH"] + assert environment["VLLM_ENABLE_PCIE_ALLREDUCE"] == "0" + assert profile["extra_vllm_args"].count("--disable-custom-all-reduce") == 1 + assert profile["extra_vllm_args"].count("--language-model-only") == 1 + assert _argument(profile, "--attention-backend") == "B12X" + assert _argument(profile, "--moe-backend") == "b12x" + assert _argument(profile, "--linear-backend") == "b12x" + compilation = json.loads(_argument(profile, "--compilation-config")) + assert compilation["pass_config"]["fuse_allreduce_rms"] is False + assert profile["identity"]["deep_ep_removed_distribution"] == ( + DEEP_EP_DISTRIBUTION + ) + assert profile["identity"]["deep_ep_module_status"] == "absent" + assert profile["identity"]["allowed_runtime_warnings"] == ( + ALLOWED_RUNTIME_WARNINGS + ) + + +def test_fastsafetensors_profile_separates_the_draft_loader() -> None: + profile = json.loads(FAST.read_text(encoding="utf-8")) + assert profile["environment"]["VLLM_FASTSAFETENSORS_QUEUE_SIZE"] == "1" + speculative = json.loads(_argument(profile, "--speculative-config")) + assert speculative["draft_load_config"] == {"load_format": "safetensors"} + assert profile["identity"]["dflash_peak_gpu_memory_status"] == "implemented" + assert profile["extra_labels"]["org.sparkring.qualification-status"] == ( + "implemented" + ) + + +def test_cuda_restore_uses_only_canonical_configuration_keys() -> None: + profile = json.loads(FAST.read_text(encoding="utf-8")) + transfer = json.loads(_argument(profile, "--kv-transfer-config")) + extra = transfer["kv_connector_extra_config"] + assert extra["spark_cache_cuda_restore"] is True + assert extra["spark_cache_cuda_placement_library"].endswith( + "libspark_cache_placement.so" + ) + assert extra["spark_cache_cuda_placement_arena_bytes"] == 256 * 1024**2 + assert extra["spark_cache_cuda_restore_io_workers"] == 8 + assert not any(key.startswith("spark_cache_native_") for key in extra) + + +def test_loader_profiles_share_model_identity_but_isolate_test_roots() -> None: + fast = json.loads(FAST.read_text(encoding="utf-8")) + safe = json.loads(SAFE.read_text(encoding="utf-8")) + fast_extra = json.loads(_argument(fast, "--kv-transfer-config"))[ + "kv_connector_extra_config" + ] + safe_extra = json.loads(_argument(safe, "--kv-transfer-config"))[ + "kv_connector_extra_config" + ] + assert fast_extra["spark_cache_draft_checkpoint_sha256"] == ( + safe_extra["spark_cache_draft_checkpoint_sha256"] + ) + assert fast_extra["spark_cache_publication_schema"] == "tail-cow-v1" + assert fast_extra["spark_cache_root"] != safe_extra["spark_cache_root"] + assert fast_extra["spark_cache_clear_once"] != safe_extra["spark_cache_clear_once"] + + +def test_resolved_profile_requires_dflash7_image_labels() -> None: + profile, _ = _resolved(FAST) + labels = profile["required_image_labels"] + assert labels["org.jovian.vllm.commit"] == VLLM_NATIVE_COMMIT + assert labels["org.sparkring.vllm.python.commit"] == VLLM_PYTHON_COMMIT + assert labels["org.jovian.b12x.commit"] == B12X_COMMIT + assert labels["org.sparkcache.deployment-profile"] == ( + "glm53-flash-dflash7-python-overlay" + ) + assert labels["org.sparkcache.cuda-config-schema"] == "canonical-v1" + assert labels["org.sparkcache.source-revision"] == SPARKCACHE_COMMIT + assert labels["org.sparkcache.source-tree"] == SPARKCACHE_TREE + assert labels["org.sparkcache.source-sha256"] == SPARKCACHE_SOURCE_SHA256 + assert labels["org.sparkring.vllm.dflash-draft-loader-patch-sha256"] == ( + DFLASH_LOADER_PATCH_SHA256 + ) + assert labels["org.sparkring.vllm.dflash-draft-loader-postimage-sha256"] == ( + DFLASH_LOADER_POSTIMAGE_SHA256 + ) + assert labels["org.sparkring.runtime.removed-deep-ep-distribution"] == ( + DEEP_EP_DISTRIBUTION + ) + assert labels["org.sparkring.runtime.deep-ep-removal-receipt-sha256"] == ( + DEEP_EP_REMOVAL_RECEIPT_SHA256 + ) + assert "adaptive" not in " ".join(labels.values()).lower() + + +def test_resolver_rejects_mtp_or_noncanonical_cuda_restore() -> None: + profile = json.loads(FAST.read_text(encoding="utf-8")) + site = yaml.safe_load(SITE.read_text(encoding="utf-8")) + kwargs = { + "image": "image", + "image_id": "sha256:" + "ab" * 32, + **DIGESTS, + } + changed = copy.deepcopy(profile) + changed["identity"]["speculator"] = "embedded_mtp" + with pytest.raises(ResolveError, match="speculator"): + resolve(changed, copy.deepcopy(site), **kwargs) + + changed = copy.deepcopy(profile) + transfer = json.loads(_argument(changed, "--kv-transfer-config")) + transfer["kv_connector_extra_config"].pop("spark_cache_cuda_restore_io_workers") + args = changed["extra_vllm_args"] + args[args.index("--kv-transfer-config") + 1] = json.dumps(transfer) + with pytest.raises(ResolveError, match="cuda_restore_io_workers"): + resolve(changed, copy.deepcopy(site), **kwargs) + + changed = copy.deepcopy(profile) + speculative = json.loads(_argument(changed, "--speculative-config")) + speculative.pop("draft_load_config") + args = changed["extra_vllm_args"] + args[args.index("--speculative-config") + 1] = json.dumps(speculative) + with pytest.raises(ResolveError, match="draft_load_config safetensors"): + resolve(changed, copy.deepcopy(site), **kwargs) + + changed = copy.deepcopy(profile) + changed["environment"]["OMP_NUM_THREADS"] = "16" + with pytest.raises(ResolveError, match="OMP_NUM_THREADS"): + resolve(changed, copy.deepcopy(site), **kwargs) + + changed = copy.deepcopy(profile) + changed["environment"].pop("VLLM_ALLREDUCE_USE_SYMM_MEM") + with pytest.raises(ResolveError, match="VLLM_ALLREDUCE_USE_SYMM_MEM"): + resolve(changed, copy.deepcopy(site), **kwargs) + + changed = copy.deepcopy(profile) + compilation = json.loads(_argument(changed, "--compilation-config")) + compilation["pass_config"]["fuse_allreduce_rms"] = True + args = changed["extra_vllm_args"] + args[args.index("--compilation-config") + 1] = json.dumps(compilation) + with pytest.raises(ResolveError, match="all-reduce RMS fusion"): + resolve(changed, copy.deepcopy(site), **kwargs) + + +def test_generic_launcher_emits_four_rank_dry_run( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + profile, site = _resolved(SAFE) + profile_path = tmp_path / "profile.json" + site_path = tmp_path / "site.yaml" + profile_path.write_text(json.dumps(profile), encoding="utf-8") + site_path.write_text(yaml.safe_dump(site, sort_keys=False), encoding="utf-8") + assert launcher.main( + ["--site", str(site_path), "--profile", str(profile_path), "plan"] + ) == 0 + plan = json.loads(capsys.readouterr().out) + assert len(plan["actions"]) == 4 + rendered = json.dumps(plan) + assert profile["container_name"] in rendered + assert "--max-num-seqs" in rendered and "32" in rendered + + +def test_quickstart_names_both_loader_statuses_and_exact_builder() -> None: + guide = GUIDE.read_text(encoding="utf-8") + assert "runtime/glm53-flash-dflash7-python-overlay/build-image.sh" in guide + assert FAST.name in guide and SAFE.name in guide + assert "implemented" in guide and "not qualified" in guide + assert DFLASH_WEIGHTS_SHA256 in guide + assert DFLASH_LOADER_PATCH_SHA256 in guide + assert "does not change the namespace" in guide + assert "PR25 compatibility profile" in guide + assert "DeepEP" in guide + assert "ModelOpt" in guide + assert "FP8 KV" in guide diff --git a/scripts/test_prepare_glm53_e105_profile.py b/scripts/test_prepare_glm53_e105_profile.py index cc9257e2..541e602d 100644 --- a/scripts/test_prepare_glm53_e105_profile.py +++ b/scripts/test_prepare_glm53_e105_profile.py @@ -96,9 +96,10 @@ def test_profile_identities_follow_model_and_speculator_semantics() -> None: profile_ids.add(profile["profile_id"]) container_names.add(profile["container_name"]) confirmations.add(profile["confirmation"]) - assert extra["spark_cache_native_restore"] is True - assert extra["spark_cache_native_arena_bytes"] == 256 * 1024**2 - assert extra["spark_cache_native_io_workers"] == 8 + assert extra["spark_cache_cuda_restore"] is True + assert extra["spark_cache_cuda_placement_arena_bytes"] == 256 * 1024**2 + assert extra["spark_cache_cuda_restore_io_workers"] == 8 + assert not any(key.startswith("spark_cache_native_") for key in extra) assert extra["spark_cache_load_threads"] == 2 assert len(identities) == 3 assert len(clear_tokens) == len(PROFILES) @@ -167,7 +168,7 @@ def test_resolver_produces_aligned_twenty_gib_profile() -> None: image_id="sha256:" + "b" * 64, parent_image="local/e105-runtime@sha256:" + "c" * 64, parent_image_id="sha256:" + "d" * 64, - native_library_sha256="e" * 64, + cuda_placement_library_sha256="e" * 64, ) assert resolved_site["runtime"]["container_image"] == resolved_profile["image"] assert resolved_site["runtime"]["container_image_digest"] == resolved_profile["image_id"] @@ -188,7 +189,7 @@ def test_each_resolved_profile_passes_site_alignment( image_id="sha256:" + "b" * 64, parent_image="local/e105-runtime@sha256:" + "c" * 64, parent_image_id="sha256:" + "d" * 64, - native_library_sha256="e" * 64, + cuda_placement_library_sha256="e" * 64, ) profile_path_out = tmp_path / "profile.json" site_path_out = tmp_path / "site.yaml" @@ -204,12 +205,12 @@ def test_each_resolved_profile_passes_site_alignment( def test_resolver_rejects_unverified_native_library() -> None: profile = json.loads(PROFILES["dflash"].read_text()) site = yaml.safe_load(SITE.read_text()) - with pytest.raises(ResolveError, match="native library"): + with pytest.raises(ResolveError, match="CUDA placement library"): resolve( profile, site, image="image", image_id="sha256:" + "a" * 64, parent_image="parent", parent_image_id="sha256:" + "b" * 64, - native_library_sha256="short", + cuda_placement_library_sha256="short", ) diff --git a/scripts/test_prepare_glm53_public_python_overlay_profile.py b/scripts/test_prepare_glm53_public_python_overlay_profile.py new file mode 100644 index 00000000..0d5a1c10 --- /dev/null +++ b/scripts/test_prepare_glm53_public_python_overlay_profile.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest +import yaml + +import sparkring_generic_launcher as launcher +from prepare_glm53_public_python_overlay_profile import ( + B12X_COMMIT, + DFLASH_LOADER_PATCH_SHA256, + DFLASH_LOADER_POSTIMAGE_SHA256, + LEASE_CONTRACT_SHA256, + MTP_CACHE_IDENTITY_SHA256, + OVERLAY_MANIFEST_SHA256, + PUBLIC_BASE, + VLLM_NATIVE_COMMIT, + VLLM_PYTHON_COMMIT, + ResolveError, + composed_mtp_identity, + resolve, +) + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ROOT / "scripts/config" +PROFILE = ( + CONFIG + / "glm53-flash-public-python-overlay-mtp5-adaptive-fastsafetensors-sparkcache-tp4-dcp1.example.json" +) +SITE = CONFIG / "glm53-flash-b12x-kda-adaptive-mtp-tp4-site.example.yaml" +IMAGE_ID = "sha256:" + "ab" * 32 +CUDA_PLACEMENT_LIBRARY = "1a" * 32 +NATIVE_ELF = "2b" * 32 +NATIVE_DISPATCH = "3c" * 32 +SOURCE_RECEIPT = "4d" * 32 + + +def _argument(profile: dict, option: str) -> str: + arguments = profile["extra_vllm_args"] + return arguments[arguments.index(option) + 1] + + +def _resolved() -> tuple[dict, dict]: + profile = json.loads(PROFILE.read_text(encoding="utf-8")) + site = yaml.safe_load(SITE.read_text(encoding="utf-8")) + profile["model_host_path"] = "/srv/models/glm53" + profile["extra_volumes"][0]["host"] = "/srv/cache/glm53-public-python-overlay" + management = ["10.20.0.10", "10.20.0.11", "10.20.0.12", "10.20.0.13"] + edge_networks = ["10.30.1.0/24", "10.30.2.0/24", "10.30.3.0/24", "10.30.4.0/24"] + ring_addresses = ( + ("10.30.1.10", "10.30.4.10"), + ("10.30.1.11", "10.30.2.11"), + ("10.30.2.12", "10.30.3.12"), + ("10.30.3.13", "10.30.4.13"), + ) + for edge, subnet in zip(site["topology"]["edges"], edge_networks, strict=True): + edge["subnet"] = subnet + for rank, address, ports in zip( + site["ranks"], management, ring_addresses, strict=True + ): + rank["ssh_target"] = f"operator@{address}" + rank["management"]["address"] = address + for port, ring_address in zip(rank["ring_ports"], ports, strict=True): + port["address"] = ring_address + for peer in rank["transport_peers"]: + peer["address"] = management[peer["rank"]] + return resolve( + profile, + site, + image="local/glm53-public-python-overlay@sha256:" + "a" * 64, + image_id=IMAGE_ID, + cuda_placement_library_sha256=CUDA_PLACEMENT_LIBRARY, + native_elf_manifest_sha256=NATIVE_ELF, + native_dispatch_manifest_sha256=NATIVE_DISPATCH, + source_receipt_sha256=SOURCE_RECEIPT, + ) + + +def test_profile_uses_distinct_composed_runtime_and_cache_identities() -> None: + profile = json.loads(PROFILE.read_text(encoding="utf-8")) + identity = profile["identity"] + assert identity["vllm_native_revision"] == VLLM_NATIVE_COMMIT + assert identity["vllm_python_revision"] == VLLM_PYTHON_COMMIT + assert identity["b12x_revision"] == B12X_COMMIT + assert identity["vllm_python_overlay_manifest_sha256"] == ( + OVERLAY_MANIFEST_SHA256 + ) + assert identity["mtp_cache_identity_sha256"] == composed_mtp_identity() + assert identity["mtp_cache_identity_sha256"] == MTP_CACHE_IDENTITY_SHA256 + assert "python-overlay" in _argument(profile, "--served-model-name") + assert "public-python-overlay" in profile["container_name"] + assert "py-0b67266-native-da4d7be" in profile["environment"]["VLLM_CACHE_ROOT"] + assert "b1d541f9" in profile["environment"]["B12X_CUTE_COMPILE_CACHE_DIR"] + + +def test_profile_selects_opaque_page_tail_copy_on_write() -> None: + profile = json.loads(PROFILE.read_text(encoding="utf-8")) + transfer = json.loads(_argument(profile, "--kv-transfer-config")) + extra = transfer["kv_connector_extra_config"] + assert extra["spark_cache_cuda_restore"] is True + assert extra["spark_cache_cuda_placement_arena_bytes"] == 256 * 1024**2 + assert extra["spark_cache_cuda_restore_io_workers"] == 8 + assert not any(key.startswith("spark_cache_native_") for key in extra) + assert extra["spark_cache_publication_schema"] == "tail-cow-v1" + assert "tail-cow" in extra["spark_cache_root"] + assert "tail-cow" in extra["spark_cache_clear_once"] + assert profile["identity"]["sparkcache_effective_publication_schema"] == ( + "page-tail-cow-v1" + ) + assert profile["extra_labels"]["org.sparkcache.publication-schema"] == ( + "tail-cow-v1" + ) + attestation = " ".join(profile["attestation_hook"]) + assert '"tail-cow-v1"' in attestation + assert "/opt/sparkring/runtime/python-overlay/sparkcache-source-tree.sha256" in ( + attestation + ) + assert "source_tree_sha256(" not in attestation + assert "/opt/sparkcache-source-identity.py" not in attestation + + +def test_resolver_requires_mixed_provenance_and_all_artifact_hashes() -> None: + profile, _ = _resolved() + labels = profile["required_image_labels"] + assert labels["org.jovian.vllm.commit"] == VLLM_NATIVE_COMMIT + assert labels["org.sparkring.vllm.python.commit"] == VLLM_PYTHON_COMMIT + assert labels["org.jovian.b12x.commit"] == B12X_COMMIT + assert labels["org.opencontainers.image.base.name"] == PUBLIC_BASE + assert labels["org.sparkring.vllm.python-overlay-manifest-sha256"] == ( + OVERLAY_MANIFEST_SHA256 + ) + assert labels["org.sparkring.vllm.dflash-draft-loader-patch-sha256"] == ( + DFLASH_LOADER_PATCH_SHA256 + ) + assert labels["org.sparkring.vllm.dflash-draft-loader-postimage-sha256"] == ( + DFLASH_LOADER_POSTIMAGE_SHA256 + ) + assert labels["org.sparkring.vllm.native-elf-manifest-sha256"] == NATIVE_ELF + assert labels["org.sparkring.vllm.native-dispatch-manifest-sha256"] == ( + NATIVE_DISPATCH + ) + assert labels["org.sparkcache.cuda-placement-library-sha256"] == ( + CUDA_PLACEMENT_LIBRARY + ) + assert "org.sparkcache.native-library-sha256" not in labels + assert labels["org.sparkcache.source-tree"] == ( + "e864ed9ad64f771188fdb59aa9738e348134d636" + ) + assert labels["org.sparkcache.vllm-contract-sha256"] == LEASE_CONTRACT_SHA256 + assert labels["org.sparkring.source-receipt-sha256"] == SOURCE_RECEIPT + assert labels["org.jovian.vllm.commit"] != labels[ + "org.sparkring.vllm.python.commit" + ] + + +def test_resolver_rejects_snapshot_publication_or_source_built_labels() -> None: + profile = json.loads(PROFILE.read_text(encoding="utf-8")) + site = yaml.safe_load(SITE.read_text(encoding="utf-8")) + arguments = { + "image": "image", + "image_id": "sha256:" + "a" * 64, + "cuda_placement_library_sha256": "b" * 64, + "native_elf_manifest_sha256": "c" * 64, + "native_dispatch_manifest_sha256": "d" * 64, + "source_receipt_sha256": "e" * 64, + } + changed = copy.deepcopy(profile) + transfer = json.loads(_argument(changed, "--kv-transfer-config")) + transfer["kv_connector_extra_config"]["spark_cache_publication_schema"] = ( + "snapshot-v1" + ) + arguments_list = changed["extra_vllm_args"] + index = arguments_list.index("--kv-transfer-config") + 1 + arguments_list[index] = json.dumps(transfer, separators=(",", ":")) + with pytest.raises(ResolveError, match="tail-cow-v1"): + resolve(changed, copy.deepcopy(site), **arguments) + + changed = copy.deepcopy(profile) + changed["required_image_labels"]["org.jovian.vllm.commit"] = VLLM_PYTHON_COMMIT + with pytest.raises(ResolveError, match="org.jovian.vllm.commit"): + resolve(changed, copy.deepcopy(site), **arguments) + + changed = copy.deepcopy(profile) + changed["attestation_hook"][2] += " && source_tree_sha256(" + with pytest.raises(ResolveError, match="clean SparkCache source receipt"): + resolve(changed, copy.deepcopy(site), **arguments) + + +def test_resolver_normalizes_legacy_placement_label_and_digest_alias() -> None: + profile = json.loads(PROFILE.read_text(encoding="utf-8")) + site = yaml.safe_load(SITE.read_text(encoding="utf-8")) + labels = profile["required_image_labels"] + labels["org.sparkcache.native-library-sha256"] = labels.pop( + "org.sparkcache.cuda-placement-library-sha256" + ) + + resolved, _ = resolve( + profile, + site, + image="image", + image_id="sha256:" + "a" * 64, + native_library_sha256="b" * 64, + native_elf_manifest_sha256="c" * 64, + native_dispatch_manifest_sha256="d" * 64, + source_receipt_sha256="e" * 64, + ) + + resolved_labels = resolved["required_image_labels"] + assert resolved_labels["org.sparkcache.cuda-placement-library-sha256"] == ( + "b" * 64 + ) + assert "org.sparkcache.native-library-sha256" not in resolved_labels + + +def test_resolver_rejects_conflicting_placement_label_aliases() -> None: + profile = json.loads(PROFILE.read_text(encoding="utf-8")) + site = yaml.safe_load(SITE.read_text(encoding="utf-8")) + profile["required_image_labels"][ + "org.sparkcache.native-library-sha256" + ] = "0" * 64 + + with pytest.raises(ResolveError, match="conflicting values"): + resolve( + profile, + site, + image="image", + image_id="sha256:" + "a" * 64, + cuda_placement_library_sha256="b" * 64, + native_elf_manifest_sha256="c" * 64, + native_dispatch_manifest_sha256="d" * 64, + source_receipt_sha256="e" * 64, + ) + + +def test_generic_launcher_builds_a_four_rank_dry_run( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + profile, site = _resolved() + profile_path = tmp_path / "profile.json" + site_path = tmp_path / "site.yaml" + profile_path.write_text(json.dumps(profile), encoding="utf-8") + site_path.write_text(yaml.safe_dump(site, sort_keys=False), encoding="utf-8") + assert launcher.main( + ["--site", str(site_path), "--profile", str(profile_path), "plan"] + ) == 0 + plan = json.loads(capsys.readouterr().out) + assert plan["command"] == "plan" + assert len(plan["actions"]) == 4 + assert {action["rank"] for action in plan["actions"]} == {0, 1, 2, 3} + rendered = json.dumps(plan) + assert profile["container_name"] in rendered + assert profile["image"] in rendered diff --git a/scripts/test_sparkcache_recipes.py b/scripts/test_sparkcache_recipes.py index d9a12686..feaff054 100644 --- a/scripts/test_sparkcache_recipes.py +++ b/scripts/test_sparkcache_recipes.py @@ -75,7 +75,8 @@ def test_compositions_pin_artifact_and_fail_closed_policy() -> None: assert recipe["serving"]["scheduler_budget_status"] == "qualified" assert recipe["sparkcache"]["kv_load_failure_policy"] == "recompute" assert recipe["sparkcache"]["streaming_snapshots"] is False - assert recipe["sparkcache"]["native_restore"] is False + assert recipe["sparkcache"]["cuda_restore"] is False + assert "native_restore" not in recipe["sparkcache"] def test_scheduler_budget_records_evidence_without_an_operator_ceiling() -> None: diff --git a/scripts/test_sparkcache_terminology.py b/scripts/test_sparkcache_terminology.py new file mode 100644 index 00000000..3b3e3476 --- /dev/null +++ b/scripts/test_sparkcache_terminology.py @@ -0,0 +1,146 @@ +"""GPU-free contracts for SparkCache CUDA configuration terminology.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from sparkcache_terminology import ( + CANONICAL_CONNECTOR_KEYS, + SparkCacheTerminologyError, + canonicalize_connector_arguments, + canonicalize_connector_extra_config, + resolve_string_alias, +) + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_every_connector_alias_normalizes_to_its_canonical_key() -> None: + legacy = { + old: index + for index, old in enumerate(CANONICAL_CONNECTOR_KEYS.values(), start=1) + } + + normalized = canonicalize_connector_extra_config(legacy) + + assert normalized == { + canonical: index + for index, canonical in enumerate(CANONICAL_CONNECTOR_KEYS, start=1) + } + + +def test_connector_alias_conflict_is_rejected_even_across_json_types() -> None: + with pytest.raises(SparkCacheTerminologyError, match="conflicting values"): + canonicalize_connector_extra_config( + { + "spark_cache_cuda_restore": True, + "spark_cache_native_restore": 1, + } + ) + + +def test_runtime_argument_normalizes_legacy_connector_json() -> None: + transfer = { + "kv_connector": "SparkContextCacheConnector", + "kv_connector_extra_config": { + "spark_cache_native_restore": True, + "spark_cache_native_library": "/opt/lib/libspark_cache_placement.so", + }, + } + + normalized = canonicalize_connector_arguments( + ("--kv-transfer-config", json.dumps(transfer)) + ) + extra = json.loads(normalized[1])["kv_connector_extra_config"] + + assert extra == { + "spark_cache_cuda_restore": True, + "spark_cache_cuda_placement_library": ( + "/opt/lib/libspark_cache_placement.so" + ), + } + + +def test_runtime_argument_rejects_conflicting_connector_json() -> None: + transfer = { + "kv_connector_extra_config": { + "spark_cache_cuda_restore": True, + "spark_cache_native_restore": False, + } + } + with pytest.raises(SparkCacheTerminologyError, match="conflicting values"): + canonicalize_connector_arguments( + ("--kv-transfer-config=" + json.dumps(transfer),) + ) + + +def test_cli_alias_accepts_one_name_or_equal_values_and_rejects_conflicts() -> None: + assert ( + resolve_string_alias( + "a" * 64, + None, + canonical_name="--cuda-placement-library-sha256", + legacy_name="--native-library-sha256", + ) + == "a" * 64 + ) + assert ( + resolve_string_alias( + "b" * 64, + "b" * 64, + canonical_name="--cuda-placement-library-sha256", + legacy_name="--native-library-sha256", + ) + == "b" * 64 + ) + with pytest.raises(SparkCacheTerminologyError, match="conflicting values"): + resolve_string_alias( + "a" * 64, + "b" * 64, + canonical_name="--cuda-placement-library-sha256", + legacy_name="--native-library-sha256", + ) + + +def test_generated_profiles_emit_only_canonical_connector_and_label_names() -> None: + forbidden = ( + "spark_cache_native_restore", + "spark_cache_native_library", + "spark_cache_native_library_sha256", + "spark_cache_native_arena_bytes", + "spark_cache_native_io_workers", + "REPLACE_WITH_NATIVE_LIBRARY_SHA256", + "org.sparkcache.native-library-sha256", + ) + profiles = tuple((ROOT / "scripts/config").glob("*sparkcache*.json")) + assert profiles + for path in profiles: + text = path.read_text(encoding="utf-8") + assert not any(name in text for name in forbidden), path.name + + +def test_operator_prose_uses_sparkcache_cuda_restore_and_placement_names() -> None: + paths = ( + ROOT / "docs/GLM53_B12X_KDA_ADAPTIVE_MTP_SPARKCACHE_TP4_QUICKSTART.md", + ROOT / "docs/GLM53_E10536A_SPARKCACHE_TP4_QUICKSTART.md", + ROOT / "recipes/sparkcache/README.md", + ROOT / "runtime/glm53-flash-adaptive-mtp-python-overlay/README.md", + ROOT / "runtime/glm53-flash-b12x-kda-adaptive-mtp/README.md", + ROOT / "scripts/config/README.md", + ) + forbidden = ( + "sparkcache native", + "native restore", + "native placement", + "native direct restore", + "native page placement", + "--native-library-sha256", + ) + text = "\n".join(path.read_text(encoding="utf-8").lower() for path in paths) + assert "sparkcache cuda restore" in text + assert "sparkcache cuda placement" in text + assert not any(phrase in text for phrase in forbidden) diff --git a/scripts/test_sparkring_generic_launcher.py b/scripts/test_sparkring_generic_launcher.py index 5fc69e44..5d3d829a 100644 --- a/scripts/test_sparkring_generic_launcher.py +++ b/scripts/test_sparkring_generic_launcher.py @@ -59,6 +59,50 @@ def test_native_profile_loads_with_declared_identity(tmp_path): assert profile.identity == {"model_revision": "b" * 40} +def test_profile_loader_normalizes_legacy_sparkcache_connector_keys(tmp_path): + transfer = { + "kv_connector": "SparkContextCacheConnector", + "kv_connector_extra_config": {"spark_cache_native_restore": True}, + } + profile = generic.load_profile( + _write_profile( + tmp_path, + "sparkcache-alias.json", + _native_document( + extra_vllm_args=["--kv-transfer-config", json.dumps(transfer)] + ), + ) + ) + + normalized = json.loads(profile.extra_vllm_args[1]) + assert normalized["kv_connector_extra_config"] == { + "spark_cache_cuda_restore": True + } + + +def test_profile_loader_rejects_conflicting_sparkcache_connector_keys(tmp_path): + transfer = { + "kv_connector": "SparkContextCacheConnector", + "kv_connector_extra_config": { + "spark_cache_cuda_restore": True, + "spark_cache_native_restore": False, + }, + } + with pytest.raises(runtime.ProfileError, match="conflicting values"): + generic.load_profile( + _write_profile( + tmp_path, + "sparkcache-conflict.json", + _native_document( + extra_vllm_args=[ + "--kv-transfer-config", + json.dumps(transfer), + ] + ), + ) + ) + + @pytest.mark.parametrize( "document, error", [