From 1b9c91f56c9f8f0120269c58e8bfe81ef862e468 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:00:21 -0500 Subject: [PATCH] Add an exact GLM DFlash7 Python-overlay runtime Construct a DFlash7-labelled image from the shared 31-file vLLM Python overlay while retaining da4d7be native extensions, B12X b1d541f, and SparkCache reconstructed-page placement at 5d571018. Prepared receipts name the external DFlash7 workload and preserve exact source, native, CUDA placement, patch, and lease-contract verification. Add TP4/DCP1 profiles for global safetensors and fastsafetensors. Both use seven DFlash tokens, draft TP4, FP8 target KV, 32 sequences, 256-token blocks, page-tail copy-on-write publication, and canonical SparkCache CUDA restore keys. Safetensors is implemented but unqualified on the composed image; fastsafetensors is research-only pending live draft-loading and peak-memory gates. Cache compatibility: the external DFlash weights digest and page-tail publication schema select a namespace distinct from embedded MTP and snapshot-v1 entries. The two loader profiles share model identity but use separate test roots and clear tokens. No wire value, digest salt, or logical geometry changes. Validation: 24 focused GPU-free tests passed; an exact prepared context verified 31 vLLM files and DFlash7 image labels; Ruff, Python compilation, JSON parsing, Bash syntax, git diff, and prose checks passed. A broader unchanged GLM README assertion remains failing at the stacked base. --- ...YTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md | 90 ++++++ docs/PREREQUISITES.md | 1 + runtime/README.md | 1 + .../README.md | 43 +++ .../build-image.sh | 125 ++++++++ .../pins.json | 144 +++++++++ .../prepare_context.py | 175 +++++++++++ .../test_dflash7_python_overlay.py | 78 +++++ .../verify_image.py | 47 +++ scripts/config/README.md | 15 + ...fetensors-sparkcache-tp4-dcp1.example.json | 197 ++++++++++++ ...fetensors-sparkcache-tp4-dcp1.example.json | 194 ++++++++++++ ...re_glm53_dflash7_python_overlay_profile.py | 283 ++++++++++++++++++ ...re_glm53_dflash7_python_overlay_profile.py | 190 ++++++++++++ 14 files changed, 1583 insertions(+) create mode 100644 docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md create mode 100644 runtime/glm53-flash-dflash7-python-overlay/README.md create mode 100644 runtime/glm53-flash-dflash7-python-overlay/build-image.sh create mode 100644 runtime/glm53-flash-dflash7-python-overlay/pins.json create mode 100644 runtime/glm53-flash-dflash7-python-overlay/prepare_context.py create mode 100644 runtime/glm53-flash-dflash7-python-overlay/test_dflash7_python_overlay.py create mode 100644 runtime/glm53-flash-dflash7-python-overlay/verify_image.py create mode 100644 scripts/config/glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json create mode 100644 scripts/config/glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json create mode 100644 scripts/prepare_glm53_dflash7_python_overlay_profile.py create mode 100644 scripts/test_prepare_glm53_dflash7_python_overlay_profile.py diff --git a/docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md b/docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md new file mode 100644 index 00000000..95e1a0f7 --- /dev/null +++ b/docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md @@ -0,0 +1,90 @@ +# Serve GLM-5.3 with external DFlash7 and the exact Python-overlay runtime + +Status: **implemented**, not qualified. The image builder, profile resolver, +and four-rank dry-run contract pass without GPUs. No image digest from this +path has completed TP4/DCP1 model loading, semantic generation, SparkCache +store/restart/restore, or concurrency qualification. + +## Runtime contract + +| Role | Exact identity | +|---|---| +| vLLM native extensions and wheel metadata | `da4d7be6c97434f6942292ed8abbf4b32dc44355` | +| vLLM Python source | `0b67266a0f37d6146a8403fb8482403c62f412d5`, tree `ba9484ccb33aa56e90ff2f447f15ca9b9da97639` | +| B12X | `b1d541f9e71a35f030d45fae437630fff7507c2a`, tree `c69cdec1c59a08e8e0e549f930fa8abcfb5134ae` | +| SparkCache reconstructed-page placement | `5d571018de5b63a9a90e5c11e6d6e86bbff4a957`, tree `e864ed9ad64f771188fdb59aa9738e348134d636` | +| Target | `local-inference-lab/GLM-5.3-Flash-NVFP4@520de24eabf507659eaef7c70f14fd584527facc` | +| External draft | `incoai/GLM-5.3-Flash-DFlash2@dc77ff1c99eeb2df044ee3d4f0094eb033fee410`, BF16 weights SHA-256 `b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b` | + +The serving contract uses seven speculative tokens, draft TP4, target FP8 KV, +32 sequences, and 256-token vLLM blocks. SparkCache selects +`tail-cow-v1`, which maps opaque GLM pages to the `page-tail-cow-v1` +namespace, and uses the canonical CUDA restore keys. + +## Build the image + +Run on Linux ARM64 from a clean checkout containing this guide: + +```bash +IMAGE='sparkring-glm53-sparkcache:dflash7-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64' \ +BUILD_RECEIPT="$PWD/glm53-dflash7-python-overlay-image-receipt.json" \ +bash runtime/glm53-flash-dflash7-python-overlay/build-image.sh +``` + +The builder verifies the public da4 image, the 31-file Python overlay, retained +native ELFs and dispatch operators, B12X, SparkCache clean source, the CUDA +placement library, four exact vLLM patches, and the eleven-file lease contract. +It does not push the image. + +## Choose the target loader + +Two profiles share the same image and DFlash7 cache identity: + +| Profile | Status | Loader behavior | +|---|---|---| +| `glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json` | **implemented**, not qualified | Uses global safetensors for target and draft. This follows the qualified-compatible loader shape but still requires live qualification on the composed 0b image. | +| `glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json` | **research-only** | Uses global fastsafetensors with queue size one. vLLM also sends DFlash loading through that global loader, and DFlash materializes all yielded GPU tensors in a dictionary. Live peak-memory and loading qualification are required. | + +Do not describe the fastsafetensors profile as qualified until the external +draft loads successfully at TP4 with measured peak GPU memory, or vLLM gains a +separately enforced draft-loader contract. + +## Resolve the profile and inspect the plan + +Copy `scripts/config/glm53-flash-tp4-site.example.yaml` +outside version control and replace every address, interface, SSH target, +device, host path, and image identity. Select one profile template: + +```bash +receipt="$PWD/glm53-dflash7-python-overlay-image-receipt.json" +image='sparkring-glm53-sparkcache:dflash7-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64' +profile_template='scripts/config/glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json' + +python scripts/prepare_glm53_dflash7_python_overlay_profile.py \ + --profile-template "$profile_template" \ + --site-template /path/to/resolved-glm53-site.yaml \ + --image "$image" \ + --image-id "$(jq -r .image_id "$receipt")" \ + --cuda-placement-library-sha256 "$(jq -r .artifacts.sparkcache_cuda_placement_sha256 "$receipt")" \ + --native-elf-manifest-sha256 "$(jq -r .runtime_contract.native_elf_manifest_sha256 "$receipt")" \ + --native-dispatch-manifest-sha256 "$(jq -r .runtime_contract.native_dispatch_manifest_sha256 "$receipt")" \ + --source-receipt-sha256 "$(jq -r .artifacts.source_receipt_sha256 "$receipt")" \ + --profile-output /path/to/glm53-dflash7-profile.json \ + --site-output /path/to/glm53-dflash7-site.yaml + +python scripts/sparkring_generic_launcher.py \ + --site /path/to/glm53-dflash7-site.yaml \ + --profile /path/to/glm53-dflash7-profile.json \ + plan +``` + +`plan` is offline. Inspect every rank action before a lifecycle command. + +## Cache namespace impact + +The external DFlash weights SHA-256 is stored as +`spark_cache_draft_checkpoint_sha256`. It cannot share entries with embedded +MTP profiles. `tail-cow-v1` also separates these entries from snapshot-v1 +manifests. The two target-loader profiles share a namespace because loader +choice does not change target or draft model state; each profile uses a +different cache root and one-shot clear token while qualification is pending. diff --git a/docs/PREREQUISITES.md b/docs/PREREQUISITES.md index ebedd273..b49e57ce 100644 --- a/docs/PREREQUISITES.md +++ b/docs/PREREQUISITES.md @@ -6,6 +6,7 @@ hardware and operator conditions required by the [GLM-5.3 Flash SparkCache quickstart](GLM53_FLASH_DFLASH2_BF16_SPARKCACHE_TP4_QUICKSTART.md), [source-built GLM-5.3 e10536a quickstart](GLM53_E10536A_SPARKCACHE_TP4_QUICKSTART.md), [GLM-5.3 adaptive-MTP and live-tensor KDA quickstart](GLM53_B12X_KDA_ADAPTIVE_MTP_SPARKCACHE_TP4_QUICKSTART.md), +[GLM-5.3 DFlash7 Python-overlay quickstart](GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md), [GLM-5.3 Flash cache-disabled quickstart](GLM53_FLASH_DFLASH2_BF16_TP4_QUICKSTART.md), [DeepSeek quickstart](DEEPSEEK_V4_FLASH_QUICKSTART.md), [Qwen3.8-27B pair quickstart](QWEN38_27B_EXL3_K5K6_PAIR_QUICKSTART.md), and diff --git a/runtime/README.md b/runtime/README.md index f9b9a7fa..b46f54fb 100644 --- a/runtime/README.md +++ b/runtime/README.md @@ -13,6 +13,7 @@ credentials, or a live-deployment result. | [`glm53-flash/`](glm53-flash/README.md) | GLM-5.3 Flash target, BF16 DFlash2, vLLM, B12X, patched NCCL, and SparkCache identity and attestation contract | | [`glm53-flash-e10536a/`](glm53-flash-e10536a/README.md) | Implemented source builder for vLLM e10536a with internal MTP5 and opt-in adaptive depth; live serving unqualified | | [`glm53-flash-b12x-kda-adaptive-mtp/`](glm53-flash-b12x-kda-adaptive-mtp/README.md) | Implemented source builder for adaptive MTP and live-tensor B12X KDA at vLLM `0b67266a`; live serving unqualified | +| [`glm53-flash-dflash7-python-overlay/`](glm53-flash-dflash7-python-overlay/README.md) | Implemented public-base builder for external DFlash7 with vLLM `0b67266a` Python over retained `da4d7be` native extensions; live serving unqualified | | [`deepseek0731-gb10/`](deepseek0731-gb10/README.md) | DeepSeek-V4-Flash-0731 GB10 parser, K5 sparse-row, and native PR431 image layer | | [`qwen38/`](qwen38/README.md) | Public-source ARM64 image builder for the Qwen3.8-27B EXL3 K5/K6 pair and cycle profiles | | [`faststart-lock.json`](faststart-lock.json) | Immutable ARM64 base-image and model-identity pins | diff --git a/runtime/glm53-flash-dflash7-python-overlay/README.md b/runtime/glm53-flash-dflash7-python-overlay/README.md new file mode 100644 index 00000000..3638b8db --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/README.md @@ -0,0 +1,43 @@ +# GLM-5.3 DFlash7 public-base Python overlay + +Status: **implemented**, not qualified. The builder constructs and verifies an +ARM64 image but no image digest from this path has completed four-rank serving +qualification. + +The image combines these exact roles: + +- retained vLLM native extensions and wheel metadata from + `da4d7be6c97434f6942292ed8abbf4b32dc44355`; +- the 31-file vLLM Python delta at + `0b67266a0f37d6146a8403fb8482403c62f412d5`; +- B12X `b1d541f9e71a35f030d45fae437630fff7507c2a`; +- SparkCache reconstructed-page placement source + `5d571018de5b63a9a90e5c11e6d6e86bbff4a957`, Git tree + `e864ed9ad64f771188fdb59aa9738e348134d636`; +- external BF16 DFlash2 weights with SHA-256 + `b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b`. + +The image builder shares the byte allowlist, retained-native verifier, exact +SparkCache patch chain, and eleven-file lease contract with +`runtime/glm53-flash-adaptive-mtp-python-overlay/`. Prepared image metadata is +rendered for external DFlash7; it does not claim adaptive MTP. + +Build on Linux ARM64: + +```bash +IMAGE='sparkring-glm53-sparkcache:dflash7-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64' \ +BUILD_RECEIPT="$PWD/glm53-dflash7-python-overlay-image-receipt.json" \ +bash runtime/glm53-flash-dflash7-python-overlay/build-image.sh +``` + +The script does not push the image. Its receipt verifies mixed vLLM +provenance, B12X, target loader dependencies, NCCL, SparkCache CUDA placement, +the clean SparkCache source receipt, and the vLLM lease contract. DFlash model +files remain operator-mounted and are verified by the runtime profile. + +Two executable profiles use external DFlash at depth seven and TP4, FP8 target +KV, 256-token vLLM blocks, 32 sequences, and SparkCache page-tail copy-on-write +publication with CUDA restore. The global safetensors profile is implemented +but unqualified on the composed 0b image. The global fastsafetensors profile is +research-only until external-draft loading and peak GPU memory pass live gates. +See `docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md`. diff --git a/runtime/glm53-flash-dflash7-python-overlay/build-image.sh b/runtime/glm53-flash-dflash7-python-overlay/build-image.sh new file mode 100644 index 00000000..d559775e --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/build-image.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Build a GLM-5.3 SparkCache image by replacing only attested Python sources. +set -euo pipefail + +here="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(git -C "${here}" rev-parse --show-toplevel)" +pins="${here}/pins.json" +shared_overlay="${repo_root}/runtime/glm53-flash-adaptive-mtp-python-overlay" +engine="${CONTAINER_ENGINE:-docker}" +image="${IMAGE:-sparkring-glm53-sparkcache:dflash7-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64}" +receipt_path="${BUILD_RECEIPT:-${PWD}/glm53-dflash7-python-overlay-image-receipt.json}" + +fatal() { + printf 'FATAL: %s\n' "$*" >&2 + exit 78 +} + +read_pin() { + python3 - "${pins}" "$1" <<'PY' +import json +import sys + +value = json.load(open(sys.argv[1], encoding="utf-8")) +for component in sys.argv[2].split("."): + value = value[int(component)] if isinstance(value, list) else value[component] +print(value) +PY +} + +tracked_inputs=( + runtime/glm53-flash-dflash7-python-overlay + runtime/glm53-flash-adaptive-mtp-python-overlay + LICENSE +) +git -C "${repo_root}" diff --quiet HEAD -- "${tracked_inputs[@]}" || + fatal "builder inputs differ from the checked-out SparkRing revision" +untracked="$(git -C "${repo_root}" ls-files --others --exclude-standard -- "${tracked_inputs[@]}")" +[[ -z "${untracked}" ]] || + fatal "builder inputs include untracked files: ${untracked%%$'\n'*}" + +public_base="$(read_pin public_base.reference)" +public_base_id="$(read_pin public_base.image_id)" +arm_builder="$(read_pin builder.arm_builder)" +vllm_native_commit="$(read_pin vllm.native_commit)" +vllm_python_commit="$(read_pin vllm.python_commit)" +vllm_python_tree="$(read_pin vllm.python_tree)" +overlay_manifest_sha256="$(read_pin vllm.overlay_manifest_sha256)" +b12x_commit="$(read_pin b12x.commit)" +b12x_tree="$(read_pin b12x.tree)" +sparkcache_commit="$(read_pin sparkcache.commit)" +sparkcache_tree="$(read_pin sparkcache.tree)" +sparkcache_source_sha256="$(read_pin sparkcache.source_tree_sha256)" +sparkring_revision="$(git -C "${repo_root}" rev-parse HEAD)" + +"${engine}" pull --platform linux/arm64 "${public_base}" +python3 "${here}/verify_image.py" \ + --engine "${engine}" --pins "${pins}" --base-image "${public_base}" >/dev/null + +workspace="$(mktemp -d)" +context="${workspace}/context" +cleanup() { + # `workspace` is created by mktemp in this process and never accepts caller input. + rm -rf -- "${workspace}" +} +trap cleanup EXIT + +python3 "${here}/prepare_context.py" \ + --repo-root "${repo_root}" "${context}" >/dev/null +python3 "${here}/prepare_context.py" --verify "${context}" >/dev/null +source_receipt_sha256="$(sha256sum "${context}/receipt.json" | cut -d' ' -f1)" +mkdir -p "${context}/base-probe" +"${engine}" run --rm --entrypoint python3 \ + --volume "${shared_overlay}:/contract:ro" \ + --volume "${pins}:/dflash-pins.json:ro" \ + --volume "${context}/base-probe:/out" \ + "${public_base}" \ + /contract/overlay_contract.py \ + --pins /dflash-pins.json \ + --manifest /contract/vllm-python-overlay.json \ + record-base \ + --site-root /usr/local/lib/python3.12/dist-packages \ + --console-script /usr/local/bin/vllm \ + --output /out/retained-native.json >/dev/null +native_elf_manifest_sha256="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8"))["native_elf_manifest_sha256"])' "${context}/base-probe/retained-native.json")" +native_dispatch_manifest_sha256="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8"))["native_dispatch_manifest_sha256"])' "${context}/base-probe/retained-native.json")" +cuda_placement_stage="sparkring-sparkcache-cuda-placement:${sparkcache_commit:0:12}-${sparkring_revision:0:12}" +"${engine}" build \ + --platform linux/arm64 \ + --target sparkcache-cuda-placement \ + --file "${context}/bundle/runtime/Containerfile" \ + --build-arg "ARM_BUILDER=${arm_builder}" \ + --tag "${cuda_placement_stage}" \ + "${context}" +sparkcache_cuda_placement_sha256="$("${engine}" run --rm --entrypoint sha256sum \ + "${cuda_placement_stage}" \ + /build/sparkcache-cuda-placement/build-cuda/libspark_cache_placement.so | cut -d' ' -f1)" + +"${engine}" build \ + --platform linux/arm64 \ + --file "${context}/bundle/runtime/Containerfile" \ + --build-arg "PUBLIC_BASE=${public_base}" \ + --build-arg "PUBLIC_BASE_ID=${public_base_id}" \ + --build-arg "ARM_BUILDER=${arm_builder}" \ + --build-arg "VLLM_NATIVE_COMMIT=${vllm_native_commit}" \ + --build-arg "VLLM_PYTHON_COMMIT=${vllm_python_commit}" \ + --build-arg "VLLM_PYTHON_TREE=${vllm_python_tree}" \ + --build-arg "B12X_COMMIT=${b12x_commit}" \ + --build-arg "B12X_TREE=${b12x_tree}" \ + --build-arg "SPARKCACHE_COMMIT=${sparkcache_commit}" \ + --build-arg "SPARKCACHE_TREE=${sparkcache_tree}" \ + --build-arg "SPARKCACHE_SOURCE_SHA256=${sparkcache_source_sha256}" \ + --build-arg "SPARKRING_REVISION=${sparkring_revision}" \ + --build-arg "SOURCE_RECEIPT_SHA256=${source_receipt_sha256}" \ + --build-arg "OVERLAY_MANIFEST_SHA256=${overlay_manifest_sha256}" \ + --build-arg "NATIVE_ELF_MANIFEST_SHA256=${native_elf_manifest_sha256}" \ + --build-arg "NATIVE_DISPATCH_MANIFEST_SHA256=${native_dispatch_manifest_sha256}" \ + --build-arg "SPARKCACHE_CUDA_PLACEMENT_SHA256=${sparkcache_cuda_placement_sha256}" \ + --tag "${image}" \ + "${context}" + +python3 "${here}/verify_image.py" \ + --engine "${engine}" --pins "${pins}" --image "${image}" \ + --output "${receipt_path}" >/dev/null +printf 'image=%s\nreceipt=%s\n' "${image}" "${receipt_path}" +"${engine}" image inspect --format '{{.Id}}' "${image}" diff --git a/runtime/glm53-flash-dflash7-python-overlay/pins.json b/runtime/glm53-flash-dflash7-python-overlay/pins.json new file mode 100644 index 00000000..a0e2d9e2 --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/pins.json @@ -0,0 +1,144 @@ +{ + "schema": "sparkring-glm53-public-python-overlay/v1", + "status": "implemented", + "qualification": "The DFlash7 image builder and offline verifier are implemented. The resulting image is unsupported for serving until an immutable digest passes four-rank TP4/DCP1 target and external-draft loading, semantic generation, SparkCache store and restore, failure-recovery, and concurrency checks.", + "platform": "linux/arm64", + "public_base": { + "reference": "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd", + "image_id": "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5", + "labels": { + "org.jovian.architecture": "linux-arm64-sm121", + "org.jovian.vllm.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.jovian.b12x.commit": "2fcf23a0ce269be27b2e03fece73d46e90e6aeea", + "org.jovian.transport": "sparkring-nccl-2.30.7-source-built", + "org.sparkring.nccl.commit": "73cf112295c33aee2b895f329f592f2a9b4b0f97", + "org.sparkring.nccl.patched-tree": "abdeb053b94c3f6d472cd55ae2b79ca821299009", + "org.sparkring.nccl.patch-sha256": "6709063fa1c25055ae77a9397dea5d89643f8211d25e7990bdd11597d08c0dde" + } + }, + "builder": { + "arm_builder": "pytorch/manylinuxaarch64-builder@sha256:f91599c49f526c77d01b68286f2bf943a5fd6a432d7e3f0afcc5784825908fe9", + "output_name": "sparkring-glm53-sparkcache:dflash7-vllm-python-0b67266-native-da4d7be-b12x-b1d541f-arm64" + }, + "vllm": { + "repository": "https://github.com/local-inference-lab/vllm.git", + "native_commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "native_tree": "4a99033352bc1a6b00852dc0e8b1cecbdd9f0ebb", + "python_commit": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "python_tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "overlay_manifest": "runtime/glm53-flash-adaptive-mtp-python-overlay/vllm-python-overlay.json", + "overlay_manifest_sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a", + "native_source_objects": { + "csrc": "9ada29088768f1bc08dadd2eed3c9738eb9ac8a1", + "cmake": "5e5bbdbe1c1b3a479656d8d6a41cc32a1982c43d", + "rust": "85c3cd52db223217d45377d3f7f884e756641de3", + "CMakeLists.txt": "bb0f51b43ef4e1c57918b551b8cf213f9059b601", + "setup.py": "ae64a13daa0f1facc255afcdb4ffcad264776b98", + "pyproject.toml": "0766645fc7481da0ec439208128b838b3348d94c", + "requirements": "d6e1c8e13cd4c4358ab422e3ef006d3f9f23e18b", + "docker/Dockerfile": "6e20f6eab482782ed90a05d91b669b59641eaa46" + }, + "retained_distribution_metadata": [ + "vllm/_version.py", + "vllm-*.dist-info", + "/usr/local/bin/vllm" + ] + }, + "b12x": { + "repository": "https://github.com/local-inference-lab/b12x.git", + "commit": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "package_version": "1.3.0", + "base_commit": "2fcf23a0ce269be27b2e03fece73d46e90e6aeea", + "base_tree": "58a046fc8faa747346f40f87166cda7e0f67ff47", + "required_caps_field": "kda_metadata_validation", + "required_caps_value": "trusted" + }, + "dependencies": { + "python": "3.12", + "torch": "2.13.0+cu130", + "cuda": "13.0.3", + "fastsafetensors": "0.3.3", + "instanttensor": "0.1.9", + "instanttensor_sdist_sha256": "d8692b97991c1a5fb2db7905b9a6ae90a7f967c7ddd853d35e41caa146750c02", + "nccl_library": "/opt/sparkring/nccl/libnccl.so.2.30.7", + "nccl_library_sha256": "5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3" + }, + "sparkcache": { + "repository": "https://github.com/FujitsuPolycom/sparkcache.git", + "commit": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "tree": "e864ed9ad64f771188fdb59aa9738e348134d636", + "source_tree_sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", + "contract": { + "path": "sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json", + "sha256": "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024", + "files": 11 + }, + "patches": [ + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/020-sparkcache-vmm-exemption.patch", + "sha256": "370b498eebf44b4e52a2d2751fa249ad4bd3d0b6fd951b063a161fb06febbe99", + "target": "vllm/config/vllm.py", + "preimage_sha256": "cc03756d9bebf2a128828e5fbf7e9766446884dfce016c34381bce0aa78bfd9e", + "postimage_sha256": "9f64f5041f7f9d953e9f6bc53de8733b3eb4035c0753056a1f646346702a0994" + }, + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/030-sparkcache-hma-load-failure.patch", + "sha256": "0202df4b5db7bd35540eebdef51fcbe2bb01845b952964080e1fe903ee19b404", + "target": "vllm/v1/core/sched/scheduler.py", + "preimage_sha256": "05c05f4b372c7a4bf76399b38b338eab657c69406b7019ab02101d2ab0c7764c", + "postimage_sha256": "337893b6b088d12eb38d8d70c866242d085134442289b4f4574a1f162c9f11c8" + }, + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/040-sparkcache-shared-prefix-lease.patch", + "sha256": "6c6d6bdc2d6e35742ef37715e88697b1f972d23b28948690c4d8d835edcaf01b", + "target": "vllm/v1/core/kv_cache_manager.py", + "preimage_sha256": "02c71da26bbac81629248ee42b0a71bd2db817d339894d647d2ff6b66fd5ad19", + "postimage_sha256": "ee03dc9ce2b720c0be6e9f572d23580ba96eff68fe3406250557e83071654af0" + }, + { + "path": "patches/vllm-glm53-b12x-kda-adaptive-mtp/041-sparkcache-shared-prefix-attach.patch", + "sha256": "b98e6bc06990f608fc5f0828c11b8eb453fbec0f8fbbf24ba45254810b7e67c3", + "target": "vllm/v1/core/sched/scheduler.py", + "preimage_sha256": "337893b6b088d12eb38d8d70c866242d085134442289b4f4574a1f162c9f11c8", + "postimage_sha256": "6d397c97f31e67a75efc01b5ddd89fa58db425de14fa43965ef2d6146b6b9bdb" + } + ] + }, + "runtime_cache_roots": { + "VLLM_CACHE_ROOT": "/cache/jit/vllm/dflash7-py-0b67266-native-da4d7be", + "B12X_CUTE_COMPILE_CACHE_DIR": "/cache/jit/b12x/b1d541f9/dflash7-cute", + "TRITON_CACHE_DIR": "/cache/jit/triton/dflash7-vllm-py-0b67266-native-da4d7be" + }, + "outputs": { + "image": null, + "image_id": null, + "image_digest": null, + "native_elf_manifest_sha256": null, + "native_dispatch_manifest_sha256": null, + "b12x_wheel_sha256": null + }, + "workload": { + "role": "external-dflash7", + "target": { + "repository": "local-inference-lab/GLM-5.3-Flash-NVFP4", + "revision": "520de24eabf507659eaef7c70f14fd584527facc", + "cache_identity_sha256": "a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9" + }, + "draft": { + "repository": "incoai/GLM-5.3-Flash-DFlash2", + "revision": "dc77ff1c99eeb2df044ee3d4f0094eb033fee410", + "config_sha256": "c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573", + "weights_sha256": "b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b", + "speculative_tokens": 7, + "tensor_parallel_size": 4 + }, + "target_loaders": { + "safetensors": "implemented", + "fastsafetensors": "research-only" + }, + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 32 + } +} diff --git a/runtime/glm53-flash-dflash7-python-overlay/prepare_context.py b/runtime/glm53-flash-dflash7-python-overlay/prepare_context.py new file mode 100644 index 00000000..74e9da66 --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/prepare_context.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Prepare the exact shared Python overlay with DFlash7 image metadata.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import shutil +from pathlib import Path +from typing import Any + + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +COMMON = ROOT / "runtime" / "glm53-flash-adaptive-mtp-python-overlay" +PINS = HERE / "pins.json" +RECEIPT_SCHEMA = "sparkring-glm53-public-python-overlay-context/v1" + + +class PrepareError(RuntimeError): + """The DFlash7 prepared context differs from its exact source contract.""" + + +def _module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise PrepareError(f"cannot load shared overlay module: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +common = _module("glm53_shared_python_overlay_prepare", COMMON / "prepare_context.py") + + +def _longpath_run(argv, *, cwd=None): + """Enable long paths in prepared Git repositories on Windows hosts.""" + + arguments = tuple(argv) + result = _original_run(arguments, cwd=cwd) + if len(arguments) >= 4 and arguments[:3] == ("git", "init", "--quiet"): + _original_run( + ("git", "-C", arguments[3], "config", "core.longpaths", "true") + ) + return result + + +_original_run = common.run + + +def _render_containerfile() -> str: + text = (COMMON / "Containerfile").read_text(encoding="utf-8") + replacements = { + "/cache/jit/vllm/py-0b67266-native-da4d7be": ( + "/cache/jit/vllm/dflash7-py-0b67266-native-da4d7be" + ), + "/cache/jit/b12x/b1d541f9/cute": ( + "/cache/jit/b12x/b1d541f9/dflash7-cute" + ), + "/cache/jit/triton/vllm-py-0b67266-native-da4d7be": ( + "/cache/jit/triton/dflash7-vllm-py-0b67266-native-da4d7be" + ), + "SparkRing GLM-5.3 adaptive-MTP Python overlay": ( + "SparkRing GLM-5.3 DFlash7 Python overlay" + ), + ( + "vLLM 0b67266 Python over da4d7be native extensions with B12X " + "b1d541f and SparkCache" + ): ( + "GLM-5.3 DFlash7 with vLLM 0b67266 Python over da4d7be native " + "extensions, B12X b1d541f, and SparkCache" + ), + "glm53-flash-adaptive-mtp-python-overlay": ( + "glm53-flash-dflash7-python-overlay" + ), + } + for old, new in replacements.items(): + if old not in text: + raise PrepareError(f"shared Containerfile omits required text: {old}") + text = text.replace(old, new) + return text + + +def _render_verify_image() -> str: + text = (COMMON / "verify_image.py").read_text(encoding="utf-8") + replacements = { + "sparkring-glm53-public-python-overlay-image/v1": ( + "sparkring-glm53-dflash7-python-overlay-image/v1" + ), + "glm53-flash-adaptive-mtp-python-overlay": ( + "glm53-flash-dflash7-python-overlay" + ), + } + for old, new in replacements.items(): + if old not in text: + raise PrepareError(f"shared image verifier omits required text: {old}") + text = text.replace(old, new) + return text + + +def _replace_runtime_files(context: Path) -> None: + runtime = context / "bundle" / "runtime" + shutil.copy2(PINS, runtime / "pins.json") + shutil.copy2(HERE / "build-image.sh", runtime / "build-image.sh") + shutil.copy2(HERE / "README.md", runtime / "README.md") + (runtime / "Containerfile").write_text( + _render_containerfile(), encoding="utf-8", newline="\n" + ) + (runtime / "verify_image.py").write_text( + _render_verify_image(), encoding="utf-8", newline="\n" + ) + receipt_path = context / "receipt.json" + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + receipt["workload"] = json.loads(PINS.read_text(encoding="utf-8"))["workload"] + for name in ( + "pins.json", + "verify_image.py", + "Containerfile", + "build-image.sh", + "README.md", + ): + relative = f"bundle/runtime/{name}" + receipt["files"][relative] = common.sha256_file(context / relative) + receipt_path.write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + + +def prepare(output: Path, *, repository_root: Path = ROOT) -> dict[str, Any]: + original_pins = common.PINS + original_run = common.run + try: + common.PINS = PINS + common.run = _longpath_run + common.prepare(output, repository_root=repository_root) + _replace_runtime_files(output) + common.verify_context(output) + return json.loads((output / "receipt.json").read_text(encoding="utf-8")) + finally: + common.PINS = original_pins + common.run = original_run + + +def verify_context(context: Path) -> dict[str, Any]: + original_pins = common.PINS + try: + common.PINS = PINS + return common.verify_context(context) + finally: + common.PINS = original_pins + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--repo-root", type=Path, default=ROOT) + parser.add_argument("--verify", action="store_true") + args = parser.parse_args() + try: + result = ( + verify_context(args.output.resolve()) + if args.verify + else prepare(args.output.resolve(), repository_root=args.repo_root.resolve()) + ) + except (OSError, KeyError, json.JSONDecodeError, PrepareError, common.PrepareError) as exc: + parser.error(str(exc)) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/runtime/glm53-flash-dflash7-python-overlay/test_dflash7_python_overlay.py b/runtime/glm53-flash-dflash7-python-overlay/test_dflash7_python_overlay.py new file mode 100644 index 00000000..ad770166 --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/test_dflash7_python_overlay.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + + +HERE = Path(__file__).resolve().parent +PINS = HERE / "pins.json" + + +def _module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +prepare = _module("glm53_dflash7_overlay_prepare", HERE / "prepare_context.py") +verify = _module("glm53_dflash7_overlay_verify", HERE / "verify_image.py") + + +def test_pins_bind_the_exact_dflash7_composition() -> None: + pins = json.loads(PINS.read_text(encoding="utf-8")) + assert pins["vllm"]["native_commit"] == ( + "da4d7be6c97434f6942292ed8abbf4b32dc44355" + ) + assert pins["vllm"]["python_commit"] == ( + "0b67266a0f37d6146a8403fb8482403c62f412d5" + ) + assert pins["b12x"]["commit"] == ( + "b1d541f9e71a35f030d45fae437630fff7507c2a" + ) + assert pins["sparkcache"]["commit"] == ( + "5d571018de5b63a9a90e5c11e6d6e86bbff4a957" + ) + workload = pins["workload"] + assert workload["role"] == "external-dflash7" + assert workload["draft"]["weights_sha256"] == ( + "b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b" + ) + assert workload["draft"]["speculative_tokens"] == 7 + assert workload["draft"]["tensor_parallel_size"] == 4 + assert workload["target_loaders"] == { + "safetensors": "implemented", + "fastsafetensors": "research-only", + } + + +def test_rendered_image_metadata_names_dflash7_not_adaptive_mtp() -> None: + recipe = prepare._render_containerfile() + assert "SparkRing GLM-5.3 DFlash7 Python overlay" in recipe + assert "glm53-flash-dflash7-python-overlay" in recipe + label_section = recipe[recipe.index("LABEL org.opencontainers.image.title=") :] + assert "adaptive-MTP" not in label_section + verifier = prepare._render_verify_image() + assert "sparkring-glm53-dflash7-python-overlay-image/v1" in verifier + assert '"glm53-flash-dflash7-python-overlay"' in verifier + + +def test_verifier_requires_the_dflash7_deployment_label() -> None: + pins = verify.shared.load_pins(PINS) + labels = verify.expected_output_labels(pins) + assert labels["org.sparkcache.deployment-profile"] == ( + "glm53-flash-dflash7-python-overlay" + ) + assert labels["org.jovian.vllm.commit"] != labels[ + "org.sparkring.vllm.python.commit" + ] + + +def test_builder_uses_the_dflash7_runtime_path_and_receipt() -> None: + script = (HERE / "build-image.sh").read_text(encoding="utf-8") + assert "runtime/glm53-flash-dflash7-python-overlay" in script + assert "dflash7-vllm-python-0b67266-native-da4d7be" in script + assert "glm53-dflash7-python-overlay-image-receipt.json" in script + assert "dflash7" in script.lower() diff --git a/runtime/glm53-flash-dflash7-python-overlay/verify_image.py b/runtime/glm53-flash-dflash7-python-overlay/verify_image.py new file mode 100644 index 00000000..771049c7 --- /dev/null +++ b/runtime/glm53-flash-dflash7-python-overlay/verify_image.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Verify a GLM-5.3 DFlash7 Python-overlay image and its exact labels.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + + +HERE = Path(__file__).resolve().parent +COMMON = HERE.parent / "glm53-flash-adaptive-mtp-python-overlay" + + +def _module(): + path = COMMON / "verify_image.py" + spec = importlib.util.spec_from_file_location("glm53_shared_overlay_verify", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load shared image verifier: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +shared = _module() +shared.PINS = HERE / "pins.json" +shared.RECEIPT_SCHEMA = "sparkring-glm53-dflash7-python-overlay-image/v1" +_shared_expected_output_labels = shared.expected_output_labels + + +def expected_output_labels(pins: dict[str, Any]) -> dict[str, str]: + labels = _shared_expected_output_labels(pins) + labels["org.sparkcache.deployment-profile"] = ( + "glm53-flash-dflash7-python-overlay" + ) + return labels + + +shared.expected_output_labels = expected_output_labels + + +def main() -> int: + return shared.main() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/config/README.md b/scripts/config/README.md index a0d041de..da42e640 100644 --- a/scripts/config/README.md +++ b/scripts/config/README.md @@ -93,6 +93,21 @@ Resolve both templates with source, contract, image, loader, adaptive-policy, and cache-identity drift. Follow the [executable quickstart](../../docs/GLM53_B12X_KDA_ADAPTIVE_MTP_SPARKCACHE_TP4_QUICKSTART.md). +## GLM-5.3 external DFlash7 Python overlay + +Status: **implemented**, not qualified. Both profiles use external DFlash at +depth seven and TP4, FP8 target KV, 256-token blocks, 32 sequences, and +SparkCache CUDA restore with page-tail copy-on-write publication. + +| File | Status | Target loader | +|---|---|---| +| `glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json` | **implemented**, not qualified | Global safetensors for target and draft | +| `glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json` | **research-only** | Global fastsafetensors queue one; external-draft peak GPU memory requires live qualification | + +Resolve either profile with +`scripts/prepare_glm53_dflash7_python_overlay_profile.py`. Follow the +[DFlash7 Python-overlay quickstart](../../docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md). + ## DeepSeek-V4-Flash-0731 **Status: implemented.** diff --git a/scripts/config/glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json b/scripts/config/glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json new file mode 100644 index 00000000..9d8dd9c5 --- /dev/null +++ b/scripts/config/glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json @@ -0,0 +1,197 @@ +{ + "schema": "sparkring-runtime-profile/v1", + "profile_id": "glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1", + "model_family": "glm53-flash", + "engine": "docker", + "container_name": "glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4", + "image": "REPLACE_WITH_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_IMAGE", + "image_id": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "model_host_path": "/REPLACE/TARGET_MODEL_HOST_PATH", + "model_container_path": "/models/target", + "shm_size": "32g", + "startup_timeout_seconds": 7200, + "environment": { + "B12X_CUTE_COMPILE_CACHE_DIR": "/cache/jit/b12x/b1d541f9/dflash7-cute", + "CMAKE_CUDA_ARCHITECTURES": "121", + "CUTE_DSL_ARCH": "sm_121a", + "FLASHINFER_CUDA_ARCH_LIST": "12.1f", + "HF_HUB_OFFLINE": "1", + "INSTANTTENSOR_BACKEND": "AIO_BUFFERED", + "INSTANTTENSOR_BUFFER_SIZE": "1268776960", + "INSTANTTENSOR_CHUNK_SIZE": "2097152", + "INSTANTTENSOR_CONCURRENCY": "1", + "INSTANTTENSOR_IO_DEPTH": "3", + "INSTANTTENSOR_MAX_FREE_MEM_USAGE": "0.05", + "LD_PRELOAD": "/opt/sparkring/nccl/libnccl.so.2", + "NCCL_ALGO": "Ring", + "NCCL_CROSS_NIC": "1", + "NCCL_CUMEM_ENABLE": "0", + "NCCL_DEBUG": "WARN", + "NCCL_IB_DISABLE": "0", + "NCCL_IB_MERGE_NICS": "0", + "NCCL_IB_SUBNET_AWARE_ROUTING": "1", + "NCCL_IGNORE_CPU_AFFINITY": "1", + "NCCL_MAX_NCHANNELS": "4", + "NCCL_MIN_NCHANNELS": "4", + "NCCL_NET": "IB", + "NCCL_NET_PLUGIN": "none", + "NCCL_P2P_LEVEL": "SYS", + "NCCL_PROTO": "LL,LL128,Simple", + "NCCL_SWITCHLESS_RING_ONLY": "1", + "OMP_NUM_THREADS": "16", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "TORCHINDUCTOR_CACHE_DIR": "/cache/jit/torchinductor/dflash7-vllm-py-0b67266-native-da4d7be", + "TORCHINDUCTOR_COMPILE_THREADS": "1", + "TORCH_CUDA_ARCH_LIST": "12.1a", + "TRANSFORMERS_OFFLINE": "1", + "TRITON_CACHE_DIR": "/cache/jit/triton/dflash7-vllm-py-0b67266-native-da4d7be", + "VLLM_B12X_MOE_FP4_FORCE_A16": "0", + "VLLM_CACHE_ROOT": "/cache/jit/vllm/dflash7-py-0b67266-native-da4d7be", + "VLLM_ENABLE_PCIE_ALLREDUCE": "0", + "VLLM_FASTSAFETENSORS_QUEUE_SIZE": "1", + "VLLM_NCCL_SO_PATH": "/opt/sparkring/nccl/libnccl.so.2", + "VLLM_NO_USAGE_STATS": "1", + "VLLM_PLUGINS": "", + "XDG_CACHE_HOME": "/cache/jit" + }, + "extra_vllm_args": [ + "--served-model-name", + "glm-5.3-flash-nvfp4-dflash7-python-overlay-0b67266-on-da4d7be-b12x-b1d541f-tp4", + "--host", + "0.0.0.0", + "--pipeline-parallel-size", + "1", + "--disable-custom-all-reduce", + "--mamba-cache-mode", + "align", + "--enable-chunked-prefill", + "--dtype", + "bfloat16", + "--kv-cache-dtype", + "fp8", + "--quantization", + "modelopt_mixed", + "--attention-backend", + "B12X", + "--block-size", + "256", + "--moe-backend", + "b12x", + "--linear-backend", + "b12x", + "--no-enable-flashinfer-autotune", + "--load-format", + "fastsafetensors", + "--enable-auto-tool-choice", + "--tool-call-parser", + "glm47", + "--reasoning-parser", + "glm45", + "--kda-prefill-backend", + "triton", + "--gpu-memory-utilization", + "0.80", + "--max-num-batched-tokens", + "8192", + "--speculative-config", + "{\"method\":\"dflash\",\"model\":\"/dflash-draft\",\"num_speculative_tokens\":7,\"draft_tensor_parallel_size\":4,\"kv_cache_dtype\":\"auto\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\"}", + "--compilation-config", + "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[8,16,32,64,128,256],\"custom_ops\":[\"all\"]}", + "--max-cudagraph-capture-size", + "256", + "--async-scheduling", + "--enable-prefix-caching", + "--cudagraph-metrics", + "--kv-transfer-config", + "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context/dflash7-py-0b67266-native-da4d7be-b12x-b1d541f-tail-cow\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_publication_schema\":\"tail-cow-v1\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_cuda_placement_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_cuda_placement_library_sha256\":\"REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256\",\"spark_cache_cuda_placement_arena_bytes\":268435456,\"spark_cache_cuda_restore_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-dflash7-python-overlay-0b67266-da4d7be-b1d541f-tail-cow-initialization\"}}" + ], + "extra_volumes": [ + { + "host": "/REPLACE/DFLASH_MODEL_HOST_PATH", + "container": "/dflash-draft", + "mode": "ro" + }, + { + "host": "/REPLACE/GLM53_DFLASH7_PYTHON_OVERLAY_CACHE_HOST_ROOT", + "container": "/cache/jit", + "mode": "rw" + } + ], + "extra_labels": { + "org.sparkring.model-profile": "glm53-flash-dflash7-python-overlay-fastsafetensors-tp4-dcp1", + "org.sparkring.external-cache": "sparkcache", + "org.sparkcache.publication-schema": "tail-cow-v1", + "org.sparkring.speculator": "external-dflash7", + "org.sparkring.qualification-status": "research-only" + }, + "init": true, + "security_opts": [ + "label=disable" + ], + "privileged": false, + "confirmation": "START_GLM53_FLASH_DFLASH7_PYTHON_OVERLAY_FASTSAFETENSORS_TP4", + "identity": { + "target_repository": "local-inference-lab/GLM-5.3-Flash-NVFP4", + "target_revision": "520de24eabf507659eaef7c70f14fd584527facc", + "target_cache_identity_sha256": "a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9", + "target_config_sha256": "676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996", + "target_weight_index_sha256": "0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb", + "speculator": "external_dflash", + "draft_repository": "incoai/GLM-5.3-Flash-DFlash2", + "draft_revision": "dc77ff1c99eeb2df044ee3d4f0094eb033fee410", + "draft_config_sha256": "c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573", + "draft_weights_sha256": "b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b", + "draft_speculative_tokens": "7", + "draft_tensor_parallel_size": "4", + "target_weight_loader": "fastsafetensors", + "target_weight_loader_queue_size": "1", + "target_weight_loader_tp_nogds": "true", + "dflash_loader_scope": "global_load_config_reaches_target_and_draft", + "dflash_peak_gpu_memory_status": "research-only", + "dflash_serving_status": "research-only", + "kv_cache_dtype": "fp8", + "vllm_block_size": "256", + "max_num_seqs": "32", + "sparkcache_publication_schema": "tail-cow-v1", + "sparkcache_effective_publication_schema": "page-tail-cow-v1", + "sparkcache_source_sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", + "sparkcache_source_revision": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "sparkcache_source_tree": "e864ed9ad64f771188fdb59aa9738e348134d636", + "vllm_native_revision": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "vllm_python_revision": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "vllm_python_tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "b12x_revision": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "b12x_tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "vllm_python_overlay_manifest_sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a" + }, + "required_image_labels": { + "org.jovian.architecture": "linux-arm64-sm121", + "org.jovian.b12x.commit": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "org.jovian.transport": "sparkring-nccl-2.30.7-source-built", + "org.jovian.vllm.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.sparkring.vllm.native.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.sparkring.vllm.python.commit": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "org.sparkring.vllm.python.tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "org.sparkring.vllm.python-overlay-manifest-sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a", + "org.sparkring.vllm.native-elf-manifest-sha256": "REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256", + "org.sparkring.vllm.native-dispatch-manifest-sha256": "REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256", + "org.sparkring.b12x.tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "org.opencontainers.image.base.name": "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd", + "org.sparkring.base.image-id": "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5", + "org.sparkcache.deployment-profile": "glm53-flash-dflash7-python-overlay", + "org.sparkcache.cuda-placement-library-sha256": "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256", + "org.sparkcache.source-revision": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "org.sparkcache.source-tree": "e864ed9ad64f771188fdb59aa9738e348134d636", + "org.sparkcache.source-sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", + "org.sparkcache.vllm-contract-sha256": "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024", + "org.sparkring.source-receipt-sha256": "REPLACE_WITH_SOURCE_RECEIPT_SHA256", + "org.sparkring.nccl.commit": "73cf112295c33aee2b895f329f592f2a9b4b0f97", + "org.sparkring.nccl.patch-sha256": "6709063fa1c25055ae77a9397dea5d89643f8211d25e7990bdd11597d08c0dde", + "org.sparkring.nccl.patched-tree": "abdeb053b94c3f6d472cd55ae2b79ca821299009" + }, + "attestation_hook": [ + "/bin/sh", + "-ec", + "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test -f /dflash-draft/config.json && test -f /dflash-draft/model.safetensors && test \"$(sha256sum /dflash-draft/config.json | cut -d ' ' -f1)\" = c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573 && test \"$(sha256sum /dflash-draft/model.safetensors | cut -d ' ' -f1)\" = b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && grep -q '\"tail-cow-v1\"' /opt/sparkcache-src/sparkcache/spark_context_cache_config.py && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/vllm-python-overlay.json | cut -d ' ' -f1)\" = e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/source-receipt.json | cut -d ' ' -f1)\" = REPLACE_WITH_SOURCE_RECEIPT_SHA256 && test \"$(python3 -c 'import json; print(json.load(open(\"/opt/sparkring/runtime/python-overlay/retained-native.json\", encoding=\"utf-8\"))[\"native_elf_manifest_sha256\"])')\" = REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256 && test \"$(python3 -c 'import json; print(json.load(open(\"/opt/sparkring/runtime/python-overlay/retained-native.json\", encoding=\"utf-8\"))[\"native_dispatch_manifest_sha256\"])')\" = REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256 && test \"$(cat /opt/sparkring/runtime/python-overlay/sparkcache-source-tree.sha256)\" = f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json | cut -d ' ' -f1)\" = 6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json" + ] +} diff --git a/scripts/config/glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json b/scripts/config/glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json new file mode 100644 index 00000000..04b21744 --- /dev/null +++ b/scripts/config/glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json @@ -0,0 +1,194 @@ +{ + "schema": "sparkring-runtime-profile/v1", + "profile_id": "glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1", + "model_family": "glm53-flash", + "engine": "docker", + "container_name": "glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4", + "image": "REPLACE_WITH_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_IMAGE", + "image_id": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "model_host_path": "/REPLACE/TARGET_MODEL_HOST_PATH", + "model_container_path": "/models/target", + "shm_size": "32g", + "startup_timeout_seconds": 7200, + "environment": { + "B12X_CUTE_COMPILE_CACHE_DIR": "/cache/jit/b12x/b1d541f9/dflash7-cute", + "CMAKE_CUDA_ARCHITECTURES": "121", + "CUTE_DSL_ARCH": "sm_121a", + "FLASHINFER_CUDA_ARCH_LIST": "12.1f", + "HF_HUB_OFFLINE": "1", + "INSTANTTENSOR_BACKEND": "AIO_BUFFERED", + "INSTANTTENSOR_BUFFER_SIZE": "1268776960", + "INSTANTTENSOR_CHUNK_SIZE": "2097152", + "INSTANTTENSOR_CONCURRENCY": "1", + "INSTANTTENSOR_IO_DEPTH": "3", + "INSTANTTENSOR_MAX_FREE_MEM_USAGE": "0.05", + "LD_PRELOAD": "/opt/sparkring/nccl/libnccl.so.2", + "NCCL_ALGO": "Ring", + "NCCL_CROSS_NIC": "1", + "NCCL_CUMEM_ENABLE": "0", + "NCCL_DEBUG": "WARN", + "NCCL_IB_DISABLE": "0", + "NCCL_IB_MERGE_NICS": "0", + "NCCL_IB_SUBNET_AWARE_ROUTING": "1", + "NCCL_IGNORE_CPU_AFFINITY": "1", + "NCCL_MAX_NCHANNELS": "4", + "NCCL_MIN_NCHANNELS": "4", + "NCCL_NET": "IB", + "NCCL_NET_PLUGIN": "none", + "NCCL_P2P_LEVEL": "SYS", + "NCCL_PROTO": "LL,LL128,Simple", + "NCCL_SWITCHLESS_RING_ONLY": "1", + "OMP_NUM_THREADS": "16", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "TORCHINDUCTOR_CACHE_DIR": "/cache/jit/torchinductor/dflash7-vllm-py-0b67266-native-da4d7be", + "TORCHINDUCTOR_COMPILE_THREADS": "1", + "TORCH_CUDA_ARCH_LIST": "12.1a", + "TRANSFORMERS_OFFLINE": "1", + "TRITON_CACHE_DIR": "/cache/jit/triton/dflash7-vllm-py-0b67266-native-da4d7be", + "VLLM_B12X_MOE_FP4_FORCE_A16": "0", + "VLLM_CACHE_ROOT": "/cache/jit/vllm/dflash7-py-0b67266-native-da4d7be", + "VLLM_ENABLE_PCIE_ALLREDUCE": "0", + "VLLM_NCCL_SO_PATH": "/opt/sparkring/nccl/libnccl.so.2", + "VLLM_NO_USAGE_STATS": "1", + "VLLM_PLUGINS": "", + "XDG_CACHE_HOME": "/cache/jit" + }, + "extra_vllm_args": [ + "--served-model-name", + "glm-5.3-flash-nvfp4-dflash7-python-overlay-0b67266-on-da4d7be-b12x-b1d541f-tp4", + "--host", + "0.0.0.0", + "--pipeline-parallel-size", + "1", + "--disable-custom-all-reduce", + "--mamba-cache-mode", + "align", + "--enable-chunked-prefill", + "--dtype", + "bfloat16", + "--kv-cache-dtype", + "fp8", + "--quantization", + "modelopt_mixed", + "--attention-backend", + "B12X", + "--block-size", + "256", + "--moe-backend", + "b12x", + "--linear-backend", + "b12x", + "--no-enable-flashinfer-autotune", + "--load-format", + "safetensors", + "--enable-auto-tool-choice", + "--tool-call-parser", + "glm47", + "--reasoning-parser", + "glm45", + "--kda-prefill-backend", + "triton", + "--gpu-memory-utilization", + "0.80", + "--max-num-batched-tokens", + "8192", + "--speculative-config", + "{\"method\":\"dflash\",\"model\":\"/dflash-draft\",\"num_speculative_tokens\":7,\"draft_tensor_parallel_size\":4,\"kv_cache_dtype\":\"auto\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"standard\"}", + "--compilation-config", + "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[8,16,32,64,128,256],\"custom_ops\":[\"all\"]}", + "--max-cudagraph-capture-size", + "256", + "--async-scheduling", + "--enable-prefix-caching", + "--cudagraph-metrics", + "--kv-transfer-config", + "{\"kv_connector\":\"SparkContextCacheConnector\",\"kv_connector_module_path\":\"sparkcache.spark_context_cache_connector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"recompute\",\"kv_connector_extra_config\":{\"spark_cache_root\":\"/cache/jit/sparkcache-context/dflash7-py-0b67266-native-da4d7be-b12x-b1d541f-safetensors-tail-cow\",\"spark_cache_model_profile\":\"glm53-flash-hybrid\",\"spark_cache_publication_schema\":\"tail-cow-v1\",\"spark_cache_target_checkpoint_sha256\":\"a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9\",\"spark_cache_draft_checkpoint_sha256\":\"b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b\",\"spark_cache_draft_policy\":\"separate\",\"spark_cache_store\":true,\"spark_cache_restore\":true,\"spark_cache_scheduler_probe\":\"none\",\"spark_cache_streaming_snapshots\":false,\"spark_cache_cuda_restore\":true,\"spark_cache_max_bytes\":51539607552,\"spark_cache_low_watermark_bytes\":42949672960,\"spark_cache_ttl_seconds\":0,\"spark_cache_min_span_tokens\":4096,\"spark_cache_max_span_tokens\":524288,\"spark_cache_cuda_placement_library\":\"/opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so\",\"spark_cache_cuda_placement_library_sha256\":\"REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256\",\"spark_cache_cuda_placement_arena_bytes\":268435456,\"spark_cache_cuda_restore_io_workers\":8,\"spark_cache_load_threads\":2,\"spark_cache_clear_once\":\"sparkring-dflash7-python-overlay-0b67266-da4d7be-b1d541f-safetensors-tail-cow-initialization\"}}" + ], + "extra_volumes": [ + { + "host": "/REPLACE/DFLASH_MODEL_HOST_PATH", + "container": "/dflash-draft", + "mode": "ro" + }, + { + "host": "/REPLACE/GLM53_DFLASH7_PYTHON_OVERLAY_CACHE_HOST_ROOT", + "container": "/cache/jit", + "mode": "rw" + } + ], + "extra_labels": { + "org.sparkring.model-profile": "glm53-flash-dflash7-python-overlay-safetensors-tp4-dcp1", + "org.sparkring.external-cache": "sparkcache", + "org.sparkcache.publication-schema": "tail-cow-v1", + "org.sparkring.speculator": "external-dflash7", + "org.sparkring.qualification-status": "implemented" + }, + "init": true, + "security_opts": [ + "label=disable" + ], + "privileged": false, + "confirmation": "START_GLM53_FLASH_DFLASH7_PYTHON_OVERLAY_SAFETENSORS_TP4", + "identity": { + "target_repository": "local-inference-lab/GLM-5.3-Flash-NVFP4", + "target_revision": "520de24eabf507659eaef7c70f14fd584527facc", + "target_cache_identity_sha256": "a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9", + "target_config_sha256": "676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996", + "target_weight_index_sha256": "0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb", + "speculator": "external_dflash", + "draft_repository": "incoai/GLM-5.3-Flash-DFlash2", + "draft_revision": "dc77ff1c99eeb2df044ee3d4f0094eb033fee410", + "draft_config_sha256": "c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573", + "draft_weights_sha256": "b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b", + "draft_speculative_tokens": "7", + "draft_tensor_parallel_size": "4", + "target_weight_loader": "safetensors", + "dflash_loader_scope": "global_load_config_reaches_target_and_draft", + "dflash_peak_gpu_memory_status": "implemented", + "dflash_serving_status": "implemented", + "kv_cache_dtype": "fp8", + "vllm_block_size": "256", + "max_num_seqs": "32", + "sparkcache_publication_schema": "tail-cow-v1", + "sparkcache_effective_publication_schema": "page-tail-cow-v1", + "sparkcache_source_sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", + "sparkcache_source_revision": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "sparkcache_source_tree": "e864ed9ad64f771188fdb59aa9738e348134d636", + "vllm_native_revision": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "vllm_python_revision": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "vllm_python_tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "b12x_revision": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "b12x_tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "vllm_python_overlay_manifest_sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a" + }, + "required_image_labels": { + "org.jovian.architecture": "linux-arm64-sm121", + "org.jovian.b12x.commit": "b1d541f9e71a35f030d45fae437630fff7507c2a", + "org.jovian.transport": "sparkring-nccl-2.30.7-source-built", + "org.jovian.vllm.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.sparkring.vllm.native.commit": "da4d7be6c97434f6942292ed8abbf4b32dc44355", + "org.sparkring.vllm.python.commit": "0b67266a0f37d6146a8403fb8482403c62f412d5", + "org.sparkring.vllm.python.tree": "ba9484ccb33aa56e90ff2f447f15ca9b9da97639", + "org.sparkring.vllm.python-overlay-manifest-sha256": "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a", + "org.sparkring.vllm.native-elf-manifest-sha256": "REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256", + "org.sparkring.vllm.native-dispatch-manifest-sha256": "REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256", + "org.sparkring.b12x.tree": "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae", + "org.opencontainers.image.base.name": "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd", + "org.sparkring.base.image-id": "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5", + "org.sparkcache.deployment-profile": "glm53-flash-dflash7-python-overlay", + "org.sparkcache.cuda-placement-library-sha256": "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256", + "org.sparkcache.source-revision": "5d571018de5b63a9a90e5c11e6d6e86bbff4a957", + "org.sparkcache.source-tree": "e864ed9ad64f771188fdb59aa9738e348134d636", + "org.sparkcache.source-sha256": "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88", + "org.sparkcache.vllm-contract-sha256": "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024", + "org.sparkring.source-receipt-sha256": "REPLACE_WITH_SOURCE_RECEIPT_SHA256", + "org.sparkring.nccl.commit": "73cf112295c33aee2b895f329f592f2a9b4b0f97", + "org.sparkring.nccl.patch-sha256": "6709063fa1c25055ae77a9397dea5d89643f8211d25e7990bdd11597d08c0dde", + "org.sparkring.nccl.patched-tree": "abdeb053b94c3f6d472cd55ae2b79ca821299009" + }, + "attestation_hook": [ + "/bin/sh", + "-ec", + "test -f /models/target/config.json && test -f /models/target/model.safetensors.index.json && test \"$(sha256sum /models/target/config.json | cut -d ' ' -f1)\" = 676382abd1e90a6c85f0c8f33d45441ecd45fd514fd7b63ce5610e732d8e4996 && test \"$(sha256sum /models/target/model.safetensors.index.json | cut -d ' ' -f1)\" = 0d1d9e6b226e76520e182de10d4e7194cc885c5cb1bf885bb90de1916ce312cb && test -f /dflash-draft/config.json && test -f /dflash-draft/model.safetensors && test \"$(sha256sum /dflash-draft/config.json | cut -d ' ' -f1)\" = c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573 && test \"$(sha256sum /dflash-draft/model.safetensors | cut -d ' ' -f1)\" = b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b && test \"$(sha256sum /opt/sparkring/nccl/libnccl.so.2 | cut -d ' ' -f1)\" = 5f1c3f10d5ace66d4ba584415bbfe42b6ac1a0a9116a3b81dcbe50516ad924b3 && grep -q '\"tail-cow-v1\"' /opt/sparkcache-src/sparkcache/spark_context_cache_config.py && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/vllm-python-overlay.json | cut -d ' ' -f1)\" = e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a && test \"$(sha256sum /opt/sparkring/runtime/python-overlay/source-receipt.json | cut -d ' ' -f1)\" = REPLACE_WITH_SOURCE_RECEIPT_SHA256 && test \"$(python3 -c 'import json; print(json.load(open(\"/opt/sparkring/runtime/python-overlay/retained-native.json\", encoding=\"utf-8\"))[\"native_elf_manifest_sha256\"])')\" = REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256 && test \"$(python3 -c 'import json; print(json.load(open(\"/opt/sparkring/runtime/python-overlay/retained-native.json\", encoding=\"utf-8\"))[\"native_dispatch_manifest_sha256\"])')\" = REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256 && test \"$(cat /opt/sparkring/runtime/python-overlay/sparkcache-source-tree.sha256)\" = f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json | cut -d ' ' -f1)\" = 6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024 && test \"$(sha256sum /opt/sparkcache-src/sparkcache/native/build-cuda/libspark_cache_placement.so | cut -d ' ' -f1)\" = REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256 && python3 /opt/sparkcache-src/sparkcache/runtime_patches/verify_lease_contract.py --vllm-root /usr/local/lib/python3.12/dist-packages --contract /opt/sparkcache-src/sparkcache/runtime_patches/vllm-kv-block-lease-contract-glm53-b12x-kda-adaptive-mtp.json" + ] +} diff --git a/scripts/prepare_glm53_dflash7_python_overlay_profile.py b/scripts/prepare_glm53_dflash7_python_overlay_profile.py new file mode 100644 index 00000000..f78bb6c2 --- /dev/null +++ b/scripts/prepare_glm53_dflash7_python_overlay_profile.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""Resolve an exact GLM-5.3 DFlash7 Python-overlay profile and TP4 site.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Any + +import yaml + + +SHA256 = re.compile(r"[0-9a-f]{64}\Z") +SHA256_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") +IMAGE_PLACEHOLDER = "REPLACE_WITH_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_IMAGE" +PLACEHOLDERS = { + "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256": "cuda_placement_library_sha256", + "REPLACE_WITH_VLLM_NATIVE_ELF_MANIFEST_SHA256": "native_elf_manifest_sha256", + "REPLACE_WITH_VLLM_NATIVE_DISPATCH_MANIFEST_SHA256": ( + "native_dispatch_manifest_sha256" + ), + "REPLACE_WITH_SOURCE_RECEIPT_SHA256": "source_receipt_sha256", +} +PUBLIC_BASE = ( + "ghcr.io/fujitsupolycom/sparkring-glm53-runtime@sha256:" + "864adfe68f458223e186a19844ac80c7adc7365e5db1f25e109b85fc19850dcd" +) +PUBLIC_BASE_ID = "sha256:7e8c0ebcb2001efb4cdab0ec9d20d53972e62db3688230044e22e61ffb1d35d5" +VLLM_NATIVE_COMMIT = "da4d7be6c97434f6942292ed8abbf4b32dc44355" +VLLM_PYTHON_COMMIT = "0b67266a0f37d6146a8403fb8482403c62f412d5" +VLLM_PYTHON_TREE = "ba9484ccb33aa56e90ff2f447f15ca9b9da97639" +B12X_COMMIT = "b1d541f9e71a35f030d45fae437630fff7507c2a" +B12X_TREE = "c69cdec1c59a08e8e0e549f930fa8abcfb5134ae" +OVERLAY_MANIFEST_SHA256 = ( + "e5e528288b173399611a4930fecc4182b7208bc1564881d52ca5d2c5c4ae0f6a" +) +SPARKCACHE_COMMIT = "5d571018de5b63a9a90e5c11e6d6e86bbff4a957" +SPARKCACHE_TREE = "e864ed9ad64f771188fdb59aa9738e348134d636" +SPARKCACHE_SOURCE_SHA256 = ( + "f7c0565521fddeff7085e4cc08043cb8d1e2bde33abc67f83b8608a162d05b88" +) +LEASE_CONTRACT_SHA256 = ( + "6defde9551cbb586fd09bb2d3020495531b6573397875a767eaae1dbad126024" +) +TARGET_IDENTITY = "a35e6bf2875c1875609b8deaec404c07c6cc80259e4222fc0b51e649498bd6b9" +DFLASH_CONFIG_SHA256 = ( + "c4aeac0101196a6e26705b34c45230bcd0c7c68ee2d2d1efdb242087f3712573" +) +DFLASH_WEIGHTS_SHA256 = ( + "b33c03475ba7322cf398828f2d8d1be376df30dc05c6b40c28c8ea8da23e410b" +) + + +class ResolveError(ValueError): + """The DFlash7 image identity or runtime profile is incomplete.""" + + +def _argument(profile: dict[str, Any], option: str) -> str: + arguments = profile.get("extra_vllm_args", []) + if arguments.count(option) != 1: + raise ResolveError(f"profile must contain exactly one {option}") + return str(arguments[arguments.index(option) + 1]) + + +def _replace(value: Any, replacements: dict[str, str]) -> Any: + if isinstance(value, str): + for old, new in replacements.items(): + value = value.replace(old, new) + return value + if isinstance(value, list): + return [_replace(item, replacements) for item in value] + if isinstance(value, dict): + return {key: _replace(item, replacements) for key, item in value.items()} + return value + + +def _require_sha256(value: str, label: str) -> None: + if SHA256.fullmatch(value) is None: + raise ResolveError(f"{label} must contain 64 lowercase hexadecimal characters") + + +def resolve( + profile: dict[str, Any], + site: dict[str, Any], + *, + image: str, + image_id: str, + cuda_placement_library_sha256: str, + native_elf_manifest_sha256: str, + native_dispatch_manifest_sha256: str, + source_receipt_sha256: str, +) -> tuple[dict[str, Any], dict[str, Any]]: + if not image or IMAGE_PLACEHOLDER in image: + raise ResolveError("DFlash7 Python-overlay image reference is unresolved") + if SHA256_ID.fullmatch(image_id) is None: + raise ResolveError("DFlash7 image ID must be sha256 plus 64 lowercase hex") + supplied = { + "cuda_placement_library_sha256": cuda_placement_library_sha256, + "native_elf_manifest_sha256": native_elf_manifest_sha256, + "native_dispatch_manifest_sha256": native_dispatch_manifest_sha256, + "source_receipt_sha256": source_receipt_sha256, + } + for name, value in supplied.items(): + _require_sha256(value, name.replace("_", " ")) + + identity = profile.get("identity", {}) + fixed_identity = { + "speculator": "external_dflash", + "draft_weights_sha256": DFLASH_WEIGHTS_SHA256, + "draft_config_sha256": DFLASH_CONFIG_SHA256, + "draft_speculative_tokens": "7", + "draft_tensor_parallel_size": "4", + "kv_cache_dtype": "fp8", + "vllm_block_size": "256", + "max_num_seqs": "32", + "sparkcache_publication_schema": "tail-cow-v1", + "sparkcache_effective_publication_schema": "page-tail-cow-v1", + "sparkcache_source_revision": SPARKCACHE_COMMIT, + "sparkcache_source_tree": SPARKCACHE_TREE, + "sparkcache_source_sha256": SPARKCACHE_SOURCE_SHA256, + "vllm_native_revision": VLLM_NATIVE_COMMIT, + "vllm_python_revision": VLLM_PYTHON_COMMIT, + "vllm_python_tree": VLLM_PYTHON_TREE, + "b12x_revision": B12X_COMMIT, + "b12x_tree": B12X_TREE, + "vllm_python_overlay_manifest_sha256": OVERLAY_MANIFEST_SHA256, + } + for name, expected in fixed_identity.items(): + if identity.get(name) != expected: + raise ResolveError(f"profile identity {name} must be {expected}") + + loader = _argument(profile, "--load-format") + if loader != identity.get("target_weight_loader"): + raise ResolveError("target loader argument and profile identity differ") + if loader == "fastsafetensors": + if profile.get("environment", {}).get("VLLM_FASTSAFETENSORS_QUEUE_SIZE") != "1": + raise ResolveError("fastsafetensors queue size must be one") + if identity.get("dflash_peak_gpu_memory_status") != "research-only": + raise ResolveError("fastsafetensors DFlash requires live peak-memory qualification") + elif loader == "safetensors": + if identity.get("dflash_peak_gpu_memory_status") != "implemented": + raise ResolveError("safetensors DFlash loader status must be implemented") + else: + raise ResolveError("DFlash7 target loader must be safetensors or fastsafetensors") + + speculative = json.loads(_argument(profile, "--speculative-config")) + expected_speculative = { + "method": "dflash", + "model": "/dflash-draft", + "num_speculative_tokens": 7, + "draft_tensor_parallel_size": 4, + } + for name, expected in expected_speculative.items(): + if speculative.get(name) != expected: + raise ResolveError(f"DFlash configuration {name} must be {expected}") + + transfer = json.loads(_argument(profile, "--kv-transfer-config")) + extra = transfer.get("kv_connector_extra_config", {}) + cuda_contract = { + "spark_cache_publication_schema": "tail-cow-v1", + "spark_cache_draft_checkpoint_sha256": DFLASH_WEIGHTS_SHA256, + "spark_cache_cuda_restore": True, + "spark_cache_cuda_placement_library": ( + "/opt/sparkcache-src/sparkcache/native/build-cuda/" + "libspark_cache_placement.so" + ), + "spark_cache_cuda_placement_library_sha256": ( + "REPLACE_WITH_CUDA_PLACEMENT_LIBRARY_SHA256" + ), + "spark_cache_cuda_placement_arena_bytes": 256 * 1024**2, + "spark_cache_cuda_restore_io_workers": 8, + "spark_cache_load_threads": 2, + } + for name, expected in cuda_contract.items(): + if extra.get(name) != expected: + raise ResolveError(f"SparkCache CUDA restore setting {name} must be {expected}") + + labels = profile.get("required_image_labels", {}) + fixed_labels = { + "org.jovian.vllm.commit": VLLM_NATIVE_COMMIT, + "org.sparkring.vllm.native.commit": VLLM_NATIVE_COMMIT, + "org.sparkring.vllm.python.commit": VLLM_PYTHON_COMMIT, + "org.sparkring.vllm.python.tree": VLLM_PYTHON_TREE, + "org.sparkring.vllm.python-overlay-manifest-sha256": OVERLAY_MANIFEST_SHA256, + "org.jovian.b12x.commit": B12X_COMMIT, + "org.sparkring.b12x.tree": B12X_TREE, + "org.opencontainers.image.base.name": PUBLIC_BASE, + "org.sparkring.base.image-id": PUBLIC_BASE_ID, + "org.sparkcache.deployment-profile": "glm53-flash-dflash7-python-overlay", + "org.sparkcache.source-revision": SPARKCACHE_COMMIT, + "org.sparkcache.source-tree": SPARKCACHE_TREE, + "org.sparkcache.source-sha256": SPARKCACHE_SOURCE_SHA256, + "org.sparkcache.vllm-contract-sha256": LEASE_CONTRACT_SHA256, + } + for name, expected in fixed_labels.items(): + if labels.get(name) != expected: + raise ResolveError(f"profile image label {name} must be {expected}") + + attestation = " ".join(str(item) for item in profile.get("attestation_hook", [])) + for required in ( + DFLASH_CONFIG_SHA256, + DFLASH_WEIGHTS_SHA256, + SPARKCACHE_SOURCE_SHA256, + LEASE_CONTRACT_SHA256, + "/opt/sparkring/runtime/python-overlay/sparkcache-source-tree.sha256", + *PLACEHOLDERS, + ): + if required not in attestation: + raise ResolveError(f"profile attestation omits {required}") + if "adaptive" in " ".join(str(value) for value in labels.values()).lower(): + raise ResolveError("DFlash7 image labels may not claim adaptive MTP") + + replacements = { + placeholder: supplied[name] for placeholder, name in PLACEHOLDERS.items() + } + profile = _replace(profile, replacements) + profile["image"] = image + profile["image_id"] = image_id + site["site"]["name"] = "glm53-flash-dflash7-python-overlay-four-rank-cycle" + site["site"]["description"] = ( + "Four DGX Spark systems serving external DFlash7 through the exact " + "vLLM Python-overlay and SparkCache CUDA restore contracts." + ) + site["runtime"]["container_image"] = image + site["runtime"]["container_image_digest"] = image_id + site["paths"]["jit_cache_dir"] = "/var/lib/sparkring/glm53-dflash7-python-overlay/jit" + site["paths"]["context_cache_dir"] = ( + "/var/lib/sparkring/glm53-dflash7-python-overlay/context" + ) + site["paths"]["evidence_dir"] = "./evidence/glm53-dflash7-python-overlay" + serving = site["serving"] + if serving["tensor_parallel_size"] != 4: + raise ResolveError("DFlash7 site tensor parallel size must be four") + if serving["decode_context_parallel_size"] != 1: + raise ResolveError("DFlash7 site DCP size must be one") + if serving["max_num_seqs"] != 32: + raise ResolveError("DFlash7 site max_num_seqs must be 32") + if "REPLACE_WITH" in json.dumps(profile): + raise ResolveError("resolved DFlash7 profile still contains a placeholder") + return profile, site + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile-template", type=Path, required=True) + parser.add_argument("--site-template", type=Path, required=True) + parser.add_argument("--image", required=True) + parser.add_argument("--image-id", required=True) + parser.add_argument("--cuda-placement-library-sha256", required=True) + parser.add_argument("--native-elf-manifest-sha256", required=True) + parser.add_argument("--native-dispatch-manifest-sha256", required=True) + parser.add_argument("--source-receipt-sha256", required=True) + parser.add_argument("--profile-output", type=Path, required=True) + parser.add_argument("--site-output", type=Path, required=True) + args = parser.parse_args() + try: + profile = json.loads(args.profile_template.read_text(encoding="utf-8")) + site = yaml.safe_load(args.site_template.read_text(encoding="utf-8")) + profile, site = resolve( + profile, + site, + image=args.image, + image_id=args.image_id, + cuda_placement_library_sha256=args.cuda_placement_library_sha256, + native_elf_manifest_sha256=args.native_elf_manifest_sha256, + native_dispatch_manifest_sha256=args.native_dispatch_manifest_sha256, + source_receipt_sha256=args.source_receipt_sha256, + ) + except (OSError, KeyError, json.JSONDecodeError, ResolveError) as exc: + parser.error(str(exc)) + args.profile_output.write_text( + json.dumps(profile, indent=2) + "\n", encoding="utf-8", newline="\n" + ) + args.site_output.write_text( + yaml.safe_dump(site, sort_keys=False), encoding="utf-8", newline="\n" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_prepare_glm53_dflash7_python_overlay_profile.py b/scripts/test_prepare_glm53_dflash7_python_overlay_profile.py new file mode 100644 index 00000000..18dd16e7 --- /dev/null +++ b/scripts/test_prepare_glm53_dflash7_python_overlay_profile.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest +import yaml + +import sparkring_generic_launcher as launcher +from prepare_glm53_dflash7_python_overlay_profile import ( + B12X_COMMIT, + DFLASH_WEIGHTS_SHA256, + VLLM_NATIVE_COMMIT, + VLLM_PYTHON_COMMIT, + ResolveError, + resolve, +) + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG = ROOT / "scripts/config" +FAST = ( + CONFIG + / "glm53-flash-dflash7-python-overlay-fastsafetensors-sparkcache-tp4-dcp1.example.json" +) +SAFE = ( + CONFIG + / "glm53-flash-dflash7-python-overlay-safetensors-sparkcache-tp4-dcp1.example.json" +) +SITE = CONFIG / "glm53-flash-tp4-site.example.yaml" +GUIDE = ROOT / "docs/GLM53_DFLASH7_PYTHON_OVERLAY_SPARKCACHE_TP4_QUICKSTART.md" +DIGESTS = { + "cuda_placement_library_sha256": "1a" * 32, + "native_elf_manifest_sha256": "2b" * 32, + "native_dispatch_manifest_sha256": "3c" * 32, + "source_receipt_sha256": "4d" * 32, +} + + +def _argument(profile: dict, option: str) -> str: + args = profile["extra_vllm_args"] + return args[args.index(option) + 1] + + +def _resolved(path: Path) -> tuple[dict, dict]: + profile = json.loads(path.read_text(encoding="utf-8")) + site = yaml.safe_load(SITE.read_text(encoding="utf-8")) + profile["model_host_path"] = "/srv/models/glm53" + profile["extra_volumes"][0]["host"] = "/srv/models/dflash2" + profile["extra_volumes"][1]["host"] = "/srv/cache/glm53-dflash7" + management = ["10.20.0.10", "10.20.0.11", "10.20.0.12", "10.20.0.13"] + networks = ["10.30.1.0/24", "10.30.2.0/24", "10.30.3.0/24", "10.30.4.0/24"] + ring = ( + ("10.30.1.10", "10.30.4.10"), + ("10.30.1.11", "10.30.2.11"), + ("10.30.2.12", "10.30.3.12"), + ("10.30.3.13", "10.30.4.13"), + ) + for edge, subnet in zip(site["topology"]["edges"], networks, strict=True): + edge["subnet"] = subnet + for rank, address, ports in zip(site["ranks"], management, ring, strict=True): + rank["ssh_target"] = f"operator@{address}" + rank["management"]["address"] = address + for port, port_address in zip(rank["ring_ports"], ports, strict=True): + port["address"] = port_address + for peer in rank["transport_peers"]: + peer["address"] = management[peer["rank"]] + return resolve( + profile, + site, + image="local/glm53-dflash7-overlay@sha256:" + "ab" * 32, + image_id="sha256:" + "cd" * 32, + **DIGESTS, + ) + + +@pytest.mark.parametrize("path,loader", [(FAST, "fastsafetensors"), (SAFE, "safetensors")]) +def test_profiles_pin_external_dflash7_and_tp4(path: Path, loader: str) -> None: + profile = json.loads(path.read_text(encoding="utf-8")) + speculative = json.loads(_argument(profile, "--speculative-config")) + assert speculative["method"] == "dflash" + assert speculative["num_speculative_tokens"] == 7 + assert speculative["draft_tensor_parallel_size"] == 4 + assert _argument(profile, "--load-format") == loader + assert profile["identity"]["draft_weights_sha256"] == DFLASH_WEIGHTS_SHA256 + assert profile["identity"]["max_num_seqs"] == "32" + assert profile["identity"]["vllm_block_size"] == "256" + assert profile["identity"]["kv_cache_dtype"] == "fp8" + + +def test_fastsafetensors_profile_is_research_only_for_dflash_peak_memory() -> None: + profile = json.loads(FAST.read_text(encoding="utf-8")) + assert profile["environment"]["VLLM_FASTSAFETENSORS_QUEUE_SIZE"] == "1" + assert profile["identity"]["dflash_peak_gpu_memory_status"] == ( + "research-only" + ) + assert profile["extra_labels"]["org.sparkring.qualification-status"] == ( + "research-only" + ) + + +def test_cuda_restore_uses_only_canonical_configuration_keys() -> None: + profile = json.loads(FAST.read_text(encoding="utf-8")) + transfer = json.loads(_argument(profile, "--kv-transfer-config")) + extra = transfer["kv_connector_extra_config"] + assert extra["spark_cache_cuda_restore"] is True + assert extra["spark_cache_cuda_placement_library"].endswith( + "libspark_cache_placement.so" + ) + assert extra["spark_cache_cuda_placement_arena_bytes"] == 256 * 1024**2 + assert extra["spark_cache_cuda_restore_io_workers"] == 8 + assert not any(key.startswith("spark_cache_native_") for key in extra) + + +def test_loader_profiles_share_model_identity_but_isolate_test_roots() -> None: + fast = json.loads(FAST.read_text(encoding="utf-8")) + safe = json.loads(SAFE.read_text(encoding="utf-8")) + fast_extra = json.loads(_argument(fast, "--kv-transfer-config"))[ + "kv_connector_extra_config" + ] + safe_extra = json.loads(_argument(safe, "--kv-transfer-config"))[ + "kv_connector_extra_config" + ] + assert fast_extra["spark_cache_draft_checkpoint_sha256"] == ( + safe_extra["spark_cache_draft_checkpoint_sha256"] + ) + assert fast_extra["spark_cache_publication_schema"] == "tail-cow-v1" + assert fast_extra["spark_cache_root"] != safe_extra["spark_cache_root"] + assert fast_extra["spark_cache_clear_once"] != safe_extra["spark_cache_clear_once"] + + +def test_resolved_profile_requires_dflash7_image_labels() -> None: + profile, _ = _resolved(FAST) + labels = profile["required_image_labels"] + assert labels["org.jovian.vllm.commit"] == VLLM_NATIVE_COMMIT + assert labels["org.sparkring.vllm.python.commit"] == VLLM_PYTHON_COMMIT + assert labels["org.jovian.b12x.commit"] == B12X_COMMIT + assert labels["org.sparkcache.deployment-profile"] == ( + "glm53-flash-dflash7-python-overlay" + ) + assert "adaptive" not in " ".join(labels.values()).lower() + + +def test_resolver_rejects_mtp_or_noncanonical_cuda_restore() -> None: + profile = json.loads(FAST.read_text(encoding="utf-8")) + site = yaml.safe_load(SITE.read_text(encoding="utf-8")) + kwargs = { + "image": "image", + "image_id": "sha256:" + "ab" * 32, + **DIGESTS, + } + changed = copy.deepcopy(profile) + changed["identity"]["speculator"] = "embedded_mtp" + with pytest.raises(ResolveError, match="speculator"): + resolve(changed, copy.deepcopy(site), **kwargs) + + changed = copy.deepcopy(profile) + transfer = json.loads(_argument(changed, "--kv-transfer-config")) + transfer["kv_connector_extra_config"].pop("spark_cache_cuda_restore_io_workers") + args = changed["extra_vllm_args"] + args[args.index("--kv-transfer-config") + 1] = json.dumps(transfer) + with pytest.raises(ResolveError, match="cuda_restore_io_workers"): + resolve(changed, copy.deepcopy(site), **kwargs) + + +def test_generic_launcher_emits_four_rank_dry_run( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + profile, site = _resolved(SAFE) + profile_path = tmp_path / "profile.json" + site_path = tmp_path / "site.yaml" + profile_path.write_text(json.dumps(profile), encoding="utf-8") + site_path.write_text(yaml.safe_dump(site, sort_keys=False), encoding="utf-8") + assert launcher.main( + ["--site", str(site_path), "--profile", str(profile_path), "plan"] + ) == 0 + plan = json.loads(capsys.readouterr().out) + assert len(plan["actions"]) == 4 + rendered = json.dumps(plan) + assert profile["container_name"] in rendered + assert "--max-num-seqs" in rendered and "32" in rendered + + +def test_quickstart_names_both_loader_statuses_and_exact_builder() -> None: + guide = GUIDE.read_text(encoding="utf-8") + assert "runtime/glm53-flash-dflash7-python-overlay/build-image.sh" in guide + assert FAST.name in guide and SAFE.name in guide + assert "research-only" in guide + assert DFLASH_WEIGHTS_SHA256 in guide