From 892e2c0dabcf6716bc1e29803b674479c1779f1a Mon Sep 17 00:00:00 2001 From: Saolence Date: Thu, 3 Sep 2026 09:47:07 +0800 Subject: [PATCH 01/16] feat(deepseek): allow SERVED_MODEL_NAME override and HF snapshot blob bind in the cycle launcher The four-Spark cycle launcher currently hardcodes --served-model-name deepseek-v4-flash-0731 and mounts only MODEL_HOST_PATH. Two operator-facing extensions, both backward compatible: * SERVED_MODEL_NAME: when set in the per-rank environment (or exported), it overrides the model name vLLM advertises and accepts; the default still reproduces recipes/deepseek-v4-flash-0731.json. The environment template documents the new key next to the other serving values, and the --check summary prints the effective name. * HF hub snapshot model directory: when MODEL_HOST_PATH points into an HF hub cache at /snapshots//, every model file is a symlink whose relative target ../../blobs/ resolves above the mounted snapshot tree. The launcher now requires the sibling blobs directory and binds it read-only at /blobs so those targets land on the real weight payloads. Plain checkpoint directories are unaffected. Offline contract tests cover the default name, the override reaching the rendered command, the snapshot blob bind, the plain-directory no-op, and the fail-closed missing-blobs path. --- .../config/deepseek-v4-flash-0731.env.example | 5 + scripts/deepseek_v4_cycle_serve.sh | 28 +++++- scripts/test_deepseek_v4_pair_launcher.py | 93 ++++++++++++++++++- 3 files changed, 124 insertions(+), 2 deletions(-) diff --git a/scripts/config/deepseek-v4-flash-0731.env.example b/scripts/config/deepseek-v4-flash-0731.env.example index e275c8ad..fe28d996 100644 --- a/scripts/config/deepseek-v4-flash-0731.env.example +++ b/scripts/config/deepseek-v4-flash-0731.env.example @@ -42,6 +42,11 @@ MAX_MODEL_LEN=1048576 MAX_NUM_SEQS=32 MAX_NUM_BATCHED_TOKENS=4096 +# SERVED_MODEL_NAME is the model name vLLM advertises and accepts in API +# requests. The default reproduces recipes/deepseek-v4-flash-0731.json; +# override it only when clients must address the service under another name. +SERVED_MODEL_NAME=deepseek-v4-flash-0731 + LD_PRELOAD=/usr/local/cuda/compat/libcuda.so.1:/opt/sparkring/nccl/libnccl.so.2 VLLM_NCCL_SO_PATH=/opt/sparkring/nccl/libnccl.so.2 TORCH_USE_RTLD_GLOBAL=1 diff --git a/scripts/deepseek_v4_cycle_serve.sh b/scripts/deepseek_v4_cycle_serve.sh index 468d541d..f62c3d91 100755 --- a/scripts/deepseek_v4_cycle_serve.sh +++ b/scripts/deepseek_v4_cycle_serve.sh @@ -138,6 +138,30 @@ esac image=ghcr.io/fujitsupolycom/gb10-vllm-serving@sha256:827a8e8c5749b78529cc0015dd174e1b19a0accc116bc142282f8b75428f98bd container_name="deepseek-v4-flash-r$NODE_RANK" model_container_path=/models/deepseek-v4-flash-0731 +served_model_name=${SERVED_MODEL_NAME:-deepseek-v4-flash-0731} + +# HuggingFace hub snapshot support: when MODEL_HOST_PATH points into an HF +# hub cache (`/snapshots//`), every model file inside is a +# symlink whose relative target ../../blobs/ resolves above the mounted +# snapshot tree. Bind the sibling blobs directory read-only so the container +# resolves those targets to the real weight payloads. Plain checkpoint +# directories are unaffected and gain no extra mount. +model_blobs_path= +case "/$MODEL_HOST_PATH/" in + */snapshots/*/) + model_blobs_path=${MODEL_HOST_PATH%/snapshots/*}/blobs + ;; + *) ;; +esac +if [ -n "$model_blobs_path" ]; then + [ -d "$model_blobs_path" ] \ + || die "HF snapshot MODEL_HOST_PATH has no sibling blobs dir: $model_blobs_path" + [ -r "$model_blobs_path" ] \ + || die "MODEL_HOST_PATH sibling blobs dir is not readable: $model_blobs_path" + blobs_mount=(-v "$model_blobs_path:/blobs:ro") +else + blobs_mount=() +fi speculative_config=$(printf \ '{"method":"dspark","num_speculative_tokens":%s,"moe_backend":"b12x"}' \ "$NUM_SPECULATIVE_TOKENS") @@ -153,6 +177,7 @@ command=( --ulimit memlock=-1:-1 --device /dev/infiniband -v "$MODEL_HOST_PATH:$model_container_path:ro" + "${blobs_mount[@]}" -v "$CACHE_HOST_PATH:/cache" --env-file "$env_file" --entrypoint /opt/venv/bin/vllm @@ -179,7 +204,7 @@ command=( --enable-auto-tool-choice --tool-call-parser deepseek_v4 --speculative-config "$speculative_config" - --served-model-name deepseek-v4-flash-0731 + --served-model-name "$served_model_name" ) if [ "$NODE_RANK" = 0 ]; then @@ -191,6 +216,7 @@ fi printf "Local rank input checks passed.\n" printf ' rank: %s\n' "$NODE_RANK" printf ' model: %s\n' "$MODEL_HOST_PATH" +printf ' served model: %s\n' "$served_model_name" printf ' cache: %s\n' "$CACHE_HOST_PATH" printf ' MAX_MODEL_LEN: %s\n' "$MAX_MODEL_LEN" printf ' MAX_NUM_SEQS: %s\n' "$MAX_NUM_SEQS" diff --git a/scripts/test_deepseek_v4_pair_launcher.py b/scripts/test_deepseek_v4_pair_launcher.py index e1308ffa..45d51cc4 100644 --- a/scripts/test_deepseek_v4_pair_launcher.py +++ b/scripts/test_deepseek_v4_pair_launcher.py @@ -11,7 +11,6 @@ import pytest - ROOT = Path(__file__).resolve().parents[1] LAUNCHER = ROOT / "scripts" / "deepseek_v4_pair_serve.sh" TEMPLATE = ROOT / "scripts" / "config" / "deepseek-v4-flash-0731-pair.env.example" @@ -152,6 +151,7 @@ def test_cycle_env_defaults_match_recipe() -> None: assert int(values["MAX_NUM_BATCHED_TOKENS"]) == serving[ "max_num_batched_tokens" ] + assert values["SERVED_MODEL_NAME"] == serving["served_model_name"] def test_launchers_pin_the_hardened_image_from_the_runtime_lock() -> None: @@ -273,3 +273,94 @@ def test_launcher_uses_host_ipc_and_16g_shm_declaration() -> None: source = LAUNCHER.read_text(encoding="utf-8") assert "--ipc host" in source assert "--shm-size 16g" in source + + +def _cycle_env_for_model(tmp_path: Path, model_dir: Path) -> Path: + """Fill the cycle template for one host model directory.""" + cache = tmp_path / "cache" + cache.mkdir() + content = CYCLE_TEMPLATE.read_text(encoding="utf-8") + replacements = { + "": "0", + "": "10.43.0.1", + "": _bash_path(model_dir), + "": _bash_path(cache), + "": "fabric0", + "": "10.43.0.1", + } + for placeholder, value in replacements.items(): + content = content.replace(placeholder, value) + env_file = tmp_path / "rank-0.env" + env_file.write_text(content, encoding="utf-8", newline="\n") + return env_file + + +def _hub_snapshot(root: Path) -> tuple[Path, Path]: + """HF hub cache layout: /snapshots/ beside /blobs.""" + snapshot = root / "snapshots" / ("a1" * 20) + blobs = root / "blobs" + snapshot.mkdir(parents=True) + blobs.mkdir() + (snapshot / "config.json").write_text("{}", encoding="utf-8") + (blobs / "payload.bin").write_bytes(b"x") + return snapshot, blobs + + +def test_cycle_served_model_name_defaults_to_recipe_name(cycle_env: Path) -> None: + result = _run_launcher(cycle_env, launcher=CYCLE_LAUNCHER) + + assert result.returncode == 0, result.stderr + assert "served model: deepseek-v4-flash-0731" in result.stdout + assert "--served-model-name deepseek-v4-flash-0731" in result.stdout + + +def test_cycle_served_model_name_override_reaches_the_command( + cycle_env: Path, +) -> None: + content = cycle_env.read_text(encoding="utf-8").replace( + "SERVED_MODEL_NAME=deepseek-v4-flash-0731", + "SERVED_MODEL_NAME=deepseek-v4-flash-alias", + ) + cycle_env.write_text(content, encoding="utf-8", newline="\n") + + result = _run_launcher(cycle_env, launcher=CYCLE_LAUNCHER) + + assert result.returncode == 0, result.stderr + assert "served model: deepseek-v4-flash-alias" in result.stdout + assert "--served-model-name deepseek-v4-flash-alias" in result.stdout + + +def test_cycle_hf_snapshot_model_binds_the_sibling_blobs_dir( + tmp_path: Path, +) -> None: + snapshot, blobs = _hub_snapshot( + tmp_path / "hub" / "models--drowzeys--keys-DeepSeekV4-Flash" + ) + env_file = _cycle_env_for_model(tmp_path, snapshot) + + result = _run_launcher(env_file, launcher=CYCLE_LAUNCHER) + + assert result.returncode == 0, result.stderr + assert f"{_bash_path(blobs)}:/blobs:ro" in result.stdout + assert _bash_path(snapshot) in result.stdout + + +def test_cycle_plain_model_dir_gains_no_blobs_mount(cycle_env: Path) -> None: + result = _run_launcher(cycle_env, launcher=CYCLE_LAUNCHER) + + assert result.returncode == 0, result.stderr + assert "/blobs:ro" not in result.stdout + + +def test_cycle_hf_snapshot_without_sibling_blobs_fails(tmp_path: Path) -> None: + snapshot = tmp_path / "hub" / "models--drowzeys--keys" / "snapshots" / ( + "a1" * 20 + ) + snapshot.mkdir(parents=True) + (snapshot / "config.json").write_text("{}", encoding="utf-8") + env_file = _cycle_env_for_model(tmp_path, snapshot) + + result = _run_launcher(env_file, launcher=CYCLE_LAUNCHER) + + assert result.returncode != 0 + assert "no sibling blobs dir" in result.stderr From ad1e2556ba39dc4ac931e91a00f7ebe820d0ef9c Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:05:13 -0500 Subject: [PATCH 02/16] Use NVFP4 MTP proposal heads in the GLM mesh profile Package the tested CUDA 13.3 and B12X compute composition with exact source verification, retain the BF16 verifier head, and isolate its persistent-cache namespace. Publish the immutable image reference with quickstart and repeated benchmark evidence. Validation: 318 CPU tests passed, 3 skipped; ARM64 image verification, exact installed-package parity, and anonymous registry access passed. --- .gitattributes | 2 + README.md | 32 +- docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md | 72 +- ...ark-mtp3-nvfp4-proposal-head-20260905.json | 74 + ...spark-mtp3-nvfp4-proposal-head-20260905.md | 122 + .../glm53-spark-mtp3-managed-mesh-tp4.json | 31 +- runtime/glm53-spark-mtp3-mesh/Dockerfile | 16 + runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md | 76 +- runtime/glm53-spark-mtp3-mesh/README.md | 76 +- runtime/glm53-spark-mtp3-mesh/build_image.py | 18 +- .../compute-image-equivalence.json | 26 + .../glm53-spark-mtp3-mesh/compute/README.md | 46 + .../compute/apply_compute.py | 167 + .../compute/prepare_compute_source.py | 155 + .../compute/source-lock.json | 59 + .../compute/test_compute.py | 125 + .../compute/verify_compute.py | 72 + .../compute/vllm-compute-files.tar.gz | Bin 0 -> 99559 bytes .../compute/vllm-e02-to-compute.patch | 4587 +++++++++++++++++ .../glm53-spark-mtp3-mesh/image-receipt.json | 44 +- runtime/glm53-spark-mtp3-mesh/pins.json | 12 +- .../glm53-spark-mtp3-mesh/public-image.json | 19 +- runtime/glm53-spark-mtp3-mesh/test_image.py | 255 + .../verify_mesh_image.py | 201 +- 24 files changed, 6178 insertions(+), 109 deletions(-) create mode 100644 performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.json create mode 100644 performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md create mode 100644 runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json create mode 100644 runtime/glm53-spark-mtp3-mesh/compute/README.md create mode 100644 runtime/glm53-spark-mtp3-mesh/compute/apply_compute.py create mode 100644 runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py create mode 100644 runtime/glm53-spark-mtp3-mesh/compute/source-lock.json create mode 100644 runtime/glm53-spark-mtp3-mesh/compute/test_compute.py create mode 100644 runtime/glm53-spark-mtp3-mesh/compute/verify_compute.py create mode 100644 runtime/glm53-spark-mtp3-mesh/compute/vllm-compute-files.tar.gz create mode 100644 runtime/glm53-spark-mtp3-mesh/compute/vllm-e02-to-compute.patch diff --git a/.gitattributes b/.gitattributes index b87812de..680ec3df 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,8 @@ *.sh text eol=lf *.patch text eol=lf runtime/** text eol=lf +runtime/glm53-spark-mtp3-mesh/compute/*.tar.gz -text +runtime/glm53-spark-mtp3-mesh/compute/*.patch whitespace=-trailing-space runtime/exl3/patches/*.patch whitespace=-trailing-space # Runtime receipts hash these source files byte-for-byte before copying them, diff --git a/README.md b/README.md index 440f09f6..72eee353 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ by that profile. | Image family / profile | Purpose | Start here | |---|---|---| | [GLM-5.3 DFlash2/SIRCL](runtime/glm53-flash-jj-r8-gb10/README.md) | Linux/ARM64 vLLM image with B12X kernels, DFlash2, SIRCL, and optional SparkCache. | [DFlash2 quickstart](docs/GLM53_JJ_R8_GB10_SPARKCACHE_TP4_QUICKSTART.md) | -| [GLM-5.3 native-MTP3 mesh](runtime/glm53-spark-mtp3-mesh/public-image.json) | Linux/ARM64 image for NVFP4-Spark, native MTP3, hybrid mesh transport, and SparkCache. | [Mesh quickstart](docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md) | +| [GLM-5.3 native-MTP3 mesh](runtime/glm53-spark-mtp3-mesh/public-image.json) | Linux/ARM64 image for NVFP4-Spark, CUDA 13.3, native MTP3 with an NVFP4/BF16 proposal head, hybrid mesh transport, and SparkCache. | [Mesh quickstart](docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md) | | [`sparkring-glm53-runtime`](https://github.com/users/FujitsuPolycom/packages/container/package/sparkring-glm53-runtime) | Pinned GLM-5.3 bases used to build serving images. | Use the digest named by the source-build guide. | | [`gb10-vllm-serving`](https://github.com/users/FujitsuPolycom/packages/container/package/gb10-vllm-serving) | Profile-specific GB10 images, including DeepSeek. | Use the image named by the selected model quickstart. | @@ -111,14 +111,24 @@ for its terms. The native-MTP3 profile below does not require these weights. | Profile | Deployment | Context | Seqs | Batch | KV / cache | Start here | |---|---|---:|---:|---:|---|---| -| NVFP4-Spark + native MTP3 + mesh · research-only | 4 Sparks · TP4/DCP4 | 1M | 16 | 8,192 | FP8 · 24 GiB/rank; SparkCache enabled | [Quickstart](docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md) | +| NVFP4-Spark + native MTP3 + NVFP4/BF16 proposal head + mesh · research-only | 4 Sparks · TP4/DCP4 | 1M | 16 | 8,192 | FP8 · 24 GiB/rank; SparkCache enabled | [Quickstart](docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md) | This profile uses the NVFP4-Spark checkpoint's built-in three-token predictor; -no external draft checkpoint is required. It combines graph-native SIRCL for +no external draft checkpoint or DFlash model is required. A separate runtime- +NVFP4 proposal head uses BF16 activations while the target/verifier head retains +its BF16 checkpoint representation. The profile combines graph-native SIRCL for selected decode shapes, dual-rail SIRCL for large prefill collectives, and RoCEnante for selected small all-reduces. Target verification captures use four-row increments through 64 rows. +The image composition uses CUDA 13.3, the native-MTP3 metadata port derived +from Local Inference Lab vLLM revision `3512b066`, and the complete B12X tree at +`b58f34ea` with vLLM integration based on `a8c796f3`. The complete B12X update +also includes MoE and dense-precision work, so comparisons against the previous +public image cannot attribute a gain to dense kernels alone. The +[head-specific comparison](performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md) +uses a control with the same metadata+dense+B12X composition. + The [profile package](runtime/glm53-spark-mtp3-mesh/README.md) provides the public image, transport files, and temperature-one warmup. The [mesh service](runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md) sets up and @@ -155,7 +165,7 @@ prompt. Each linked record gives its sampling, cache settings, and repeat counts | Profile | Decode context | Prefill | C1 decode | C8 decode | Highest C at this context | Coding peak | |---|---:|---:|---:|---:|---:|---:| -| [GLM-5.3 NVFP4-Spark · native MTP3 + mesh · 4 Sparks](performance/records/glm53-flash/spark-mtp3-mesh-20260905.md) | 8K | 2,703 (8K scout) | 48.2 | 168.8 | C16: 231.3 | — | +| [GLM-5.3 NVFP4-Spark · native MTP3 + NVFP4/BF16 proposal head + mesh · 4 Sparks](performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md) | 8K | 2,670 (8K scout mean) | 51.6 | 168.8 | C8: 168.8 | — | | [GLM-5.3 NVFP4-Spark · DFlash2 exact request-batch graphs · 4 Sparks](performance/records/glm53-flash/dflash2-exact-concurrency-graphs-20260904.md) | 16K | 2,717 (16K scout) | 43.05 | 134.3 | C16: 187.0 | — | | [GLM-5.3 NVFP4 · DFlash2/B12X-KDA DCP4 · 4 Sparks](performance/records/glm53-flash/b12x-kda-dcp4-20260903.md) | 16K | 2,649 (16K scout) | 37.97 | — | C4: 90.36 | — | | [GLM-5.2 EXL3 3.5-bpw · 4 Sparks](performance/records/glm-3.5bpw/normalized-base-20260822.md) | 16K | 671 (16K) | 20.15 | 64.13 | C8: 64.13 | 25.39 | @@ -164,11 +174,15 @@ prompt. Each linked record gives its sampling, cache settings, and repeat counts | [Qwen3.8-27B EXL3 K5/K6 · 2 Sparks](performance/records/qwen38-27b/normalized-tp2-1m-probmtp-temp1-20260823.md) | 16K | 1,367 (16K) | 29.50 | 142.20 | C16: 184.39 | 39.95 | | [Qwen3.8-27B EXL3 K5/K6 · 4 Sparks](performance/records/qwen38-27b/normalized-tp4-1m-probmtp-temp1-20260823.md) | 16K | 1,964 (16K) | 35.07 | 191.02 | C8: 191.02 | 48.46 | -The native-MTP3 mesh row uses one observation per cell with caching enabled. -Its [consolidated report](performance/records/glm53-flash/spark-mtp3-validation-summary-20260905.md) -also includes three-pass cold-prefix prefill results, the 32K/64K decode -matrices, Estonia **30/30** at C8, and **4/4** needle-hunt checks through -507,367 prompt tokens. +The native-MTP3 proposal-head row uses three observations per decode cell and +three prefill scouts per listed context. Relative to two shared-BF16-head +controls with the same compute composition, C1 changed by +8.22% raw output +throughput and +4.90% normalized sequence steps/s; C2/C4/C8 were mixed and +prefill was flat within 0.36%. The previous image's +[consolidated report](performance/records/glm53-flash/spark-mtp3-validation-summary-20260905.md) +retains the 32K/64K matrix, Estonia **30/30** at C8, and **4/4** needle-hunt +checks through 507,367 prompt tokens, with the image identity recorded for +each test. See [benchmark results and throughput tables](docs/RESULTS.md) for full matrices, sample counts, exact settings, and limitations. diff --git a/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md b/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md index 678f4315..bb5c9e89 100644 --- a/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md +++ b/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md @@ -12,6 +12,10 @@ The [public application-install record](../performance/records/glm53-flash/spark covers fresh public checkouts, extracted image artifacts, empty application caches, installation, native correctness, and model-restart cache restoration on four prepared hosts. It does not qualify a factory-reset OS/network setup. +Both functional records are image-specific to the earlier published +composition. The NVFP4/BF16 proposal-head performance record does not transfer +their restart, cache, or failure-containment qualification to the current +public image ID; repeat those checks before describing it as qualified. **Starting with four stock Sparks and no image?** Follow [the managed-mesh prerequisite section](PREREQUISITES.md#four-spark-managed-hardware-forwarded-mesh) @@ -20,15 +24,18 @@ data interfaces, GID/MTU checks, and driver configuration required below. Return here to pull the published image and deploy the model. The profile uses the `GLM-5.3-Flash-NVFP4-Spark` target's built-in multi-token -predictor with three speculative tokens. Graph-native SIRCL handles most +predictor with three speculative tokens. Its separate proposal head is packed +to NVFP4 at model load and uses BF16 activations. The target/verifier head +retains its BF16 checkpoint representation. Graph-native SIRCL handles most captured target verification, fused SIRCL handles large eager prefill, and RoCEnante handles selected small all-reduces. Patched NCCL retains the other collectives. The host fabric supplies hardware-forwarded paths between -opposite ranks without extra diagonal cables. +opposite ranks without extra diagonal cables. No external draft checkpoint or +DFlash model is used. The [profile contract](../runtime/glm53-spark-mtp3-mesh/README.md) and [pins](../runtime/glm53-spark-mtp3-mesh/pins.json) are the canonical inputs. -The [throughput record](../performance/records/glm53-flash/spark-mtp3-mesh-20260905.md) +The [proposal-head throughput record](../performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md) reports observations, not a general performance guarantee. ## Attribution and design origins @@ -46,16 +53,28 @@ managed deployment. It does not claim to originate RoCEnante or install both complete PRs unchanged. The [vendored-source provenance](../third_party/b12x_roce/README.md) identifies the included code and retained license. +The proposal-head and metadata implementation is derived from +[Local Inference Lab vLLM revision `3512b066`](https://github.com/local-inference-lab/vllm/commit/3512b066e7796128c0c380ccc558182960f2f0ea), +as retained in +[revision `a8c796f3`](https://github.com/local-inference-lab/vllm/commit/a8c796f3af74106b2d8d441e9ec54588936a5388). +The packaged compute source uses the complete Local Inference Lab B12X tree at +[revision `b58f34ea`](https://github.com/local-inference-lab/b12x/commit/b58f34eaf978277621efced6678e6713fd7122e4). +That tree includes MoE and +dense-precision changes in addition to the head kernel. SparkRing does not +claim an isolated dense-kernel result for this composition. + ## Recorded benchmark observations See the [consolidated validation report](../performance/records/glm53-flash/spark-mtp3-validation-summary-20260905.md) for completed checks, repeat counts, and the remaining test plan. The [profile results table](../runtime/glm53-spark-mtp3-mesh/README.md#operator-benchmark-observations) -shows the full concurrency matrix: aggregate decode reached **231.3 tok/s at -8K/C16**, and concurrency-one prefill scouts measured **2,703–2,787 prompt -tokens/s** across 8K–128K contexts. The linked record provides the measured -configuration, sampling settings, and single-run measurement conditions. +shows the completed three-run C1/C2/C4/C8 screen. At 8K, aggregate decode means +were **51.6, 76.9, 120.8, and 168.8 tok/s**. Against two shared-BF16-head +controls using the same metadata+dense+B12X base, C1 improved **8.22% raw** and +**4.90% in normalized sequence steps/s**. Higher concurrency was mixed and +prefill means were flat within 0.36% over 8K–128K. The linked record provides +the receipt hashes, exact settings, and limitations. A separate [Estonia long-context accuracy benchmark](../performance/records/glm53-flash/spark-mtp3-country-recall-20260905.md) completed **30/30 correct answers at C8** on one repeated 133,208-token prompt, @@ -70,7 +89,7 @@ assume a Linux Bash shell and a checkout containing this guide: ```bash set -euo pipefail -git clone --branch codex/glm53-spark-mtp3-mesh https://github.com/FujitsuPolycom/sparkring.git +git clone --branch main https://github.com/FujitsuPolycom/sparkring.git cd sparkring git rev-parse HEAD test -f runtime/glm53-spark-mtp3-mesh/managed_install.py @@ -293,6 +312,15 @@ This addresses the greedy-only warmup gap tracked in Completed warmup establishes that its requests ran, not comprehensive sampling correctness or thinking-enabled generation coverage. +The pinned image supplies the defaults +`VLLM_MTP_NVFP4_LM_HEAD=1`, `VLLM_LM_HEAD_A16=1`, and +`VLLM_MXFP8_LM_HEAD=0`. The target checkpoint's unquantized `lm_head.weight` +initializes a distinct proposal-head copy on each tensor-parallel rank. For +154,880 vocabulary rows, width 4,096, and TP4, the packed NVFP4 values and +scales add approximately 85.08 MiB per rank. The retained BF16 target head +remains allocated, so 85.08 MiB is an added proposal allocation, not a net +model-memory reduction. + For a full MTP3 verification batch, target rows are approximately `Q = 4 × active requests`. Draft execution and partial batches can use different shapes. A capture list is not proof that every live step replays a @@ -333,12 +361,16 @@ uses eight I/O workers, eight load threads, eight pending operations, and two separate from SIRCL's two 64 MiB transport arenas. Native MTP's cache draft identity is the target checkpoint. The profile uses -the dedicated namespace in `pins.json`; external-draft-tagged entries must -not be renamed into it. The `draft_policy=separate` field describes cache -registration layout, not an external draft model. The linked functional record -includes an uncached publication and stopped-container restoration under this -identity. It covers one recall prompt and does not qualify other checkpoints, -all context lengths, or concurrent cache workloads. +the dedicated namespace +`glm53-spark-df116c4f-mtp3-nvfp4-a16-b58f34ea-mesh4204fabc-tail-cow-v2`; +shared-BF16-head and external-DFlash entries must not be renamed into it. The +`draft_policy=separate` field describes cache registration layout, not an +external draft model. The linked functional record +includes an uncached publication and stopped-container restoration under the +previous image's identity. Persistent restoration under the new namespace is +unqualified until the same stopped-container check passes. The earlier record +covers one recall prompt and does not qualify other checkpoints, all context +lengths, or concurrent cache workloads. ## Obtain the image and target @@ -350,9 +382,17 @@ input accepted by the renderer, installer, and native qualification runner. Keep both with the checkout; do not substitute `public-image.json` for the content receipt. +The content and registry receipts identify the same public image. The +[compute-image equivalence record](../runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json) +verifies all 4,891 vLLM, 385 B12X, and 150 SparkCache package files plus the +selected environment against tested private image +`sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74`. +That proves build/content equivalence, not a fresh serving, restart, or +persistent-cache qualification for the public image ID. + ```bash -mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:23f00af873ccc784cfb742b7be2a29c6d3c20ebec9741843c025320bb9c04685' -mtp_image_id='sha256:26273b8e358df139ae913610a5d43084ff0fd08aafe282ef633a3bc74afefe47' +mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86' +mtp_image_id='sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df' docker pull "${mtp_image}" test "$(docker image inspect "${mtp_image}" --format '{{.Id}}')" = "${mtp_image_id}" diff --git a/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.json b/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.json new file mode 100644 index 00000000..eadc6683 --- /dev/null +++ b/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.json @@ -0,0 +1,74 @@ +{ + "schema": "sparkring-saved-proposal-head-comparison/v1", + "status": "research-only", + "conditions": { + "hardware": "4 NVIDIA DGX Spark GB10 systems", + "topology": "TP4/DCP4/PP1 direct-cycle mesh", + "model": "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark@df116c4fb16b1d37ae43d2cfd624de26ffbc832e", + "speculation": "native MTP3; no external DFlash model", + "sampling": {"temperature": 1.0, "ignore_eos": true}, + "measurement_seconds": 20.0, + "decode_context_tokens": 8192, + "concurrency": [1, 2, 4, 8], + "proposal_head": "runtime NVFP4 weights with BF16 activations", + "target_verifier_head": "retained BF16 checkpoint representation", + "cuda_toolchain": "13.3.33", + "b12x_revision": "b58f34eaf978277621efced6678e6713fd7122e4", + "vllm_head_donor_revision": "3512b066e7796128c0c380ccc558182960f2f0ea", + "vllm_complete_donor_revision": "a8c796f3af74106b2d8d441e9ec54588936a5388", + "metadata_dense_base_image_id": "sha256:f35ed3d1df1ee57f66ba571a491d6dd5575d1a2e619cbc63d065008894777ffe", + "tested_private_image_id": "sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74", + "published_image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86", + "published_image_id": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", + "published_source_lock_sha256": "2a444f7c1ad4319f64afbffb16403491a4863934372cf818f18c510d58c0d00e", + "published_compute_equivalence": "all 4891 vLLM, 385 B12X, and 150 SparkCache package files plus selected environment match the tested private image", + "published_exact_image_serving_repeated": false, + "tested_image_receipt_sha256": "aa8f1bf3d892af7b0a8059626ec91a83e2dbc5a39c8d58c0fb82c614e88dbaba", + "model_ready_receipt_sha256": "fe786299a6459cd57dcf5362591b56d94569dd473b0d1fefb47ca374399983f3", + "live_control_receipt_sha256": "b0ebcf65be43998791f66da819ebaa98aa4b4323b9b76cc1e363d977750e3341", + "proposal_head_manifest_sha256": "9408c82bcb5a47ce0cb580d6835b23924b1d40a29352ac97f72d2fd8a77b9fb1", + "final_private_image_identity_available_in_receipts": true + }, + "receipts": { + "proposal_head": [ + { + "basename": "glm-5.3-flash-spark-dcp4-MTP3-SIRCL-and-MESH-r2420260905-122857.json", + "sha256": "05d846465153230f4312a59aefef12c499d81634855f3c84cb6757baf653c0e4" + }, + { + "basename": "glm-5.3-flash-spark-dcp4-MTP3-SIRCL-and-MESH-r2420260905-123541.json", + "sha256": "9664bc0410db423d3edc759106f10bc24a0fe1d15952102cdd74913ff96c2149" + }, + { + "basename": "glm-5.3-flash-spark-dcp4-MTP3-SIRCL-and-MESH-r2420260905-124222.json", + "sha256": "4213df0fc47ba53308ea7dc076e3e1c77e9097e5eaef95db4e09ca4cd404e434" + } + ], + "shared_bf16_proposal_head_control": [ + { + "basename": "glm-5.3-flash-spark-dcp4-MTP3-SIRCL-and-MESH-r2420260905-114306.json", + "sha256": "f7ae32cef4c9ea606477a28ff0465403e78f9519e2d771e4af0f13017b671677" + }, + { + "basename": "glm-5.3-flash-spark-dcp4-MTP3-SIRCL-and-MESH-r2420260905-115629.json", + "sha256": "7c3ff3210e805b8a801d4e8ea49152d0545a982707cc9010a174f77ff3d4e673" + } + ], + "published_context_only": { + "basename": "glm-5.3-flash-spark-dcp4-dflash7-bf16-SIRCL-and-MESH-20260904-233405.json", + "sha256": "f0916f6b72cb8256225169b44c4f11e3ca764a5dd854977b8963686197b843fa" + } + }, + "decode_8k": [ + {"concurrency": 1, "control_raw_tps": [48.839071256871314, 46.61270983199115], "proposal_raw_tps": [49.35, 52.7922337868845, 52.8], "control_steps_per_s": [18.014411529173845, 17.93565147876187], "proposal_steps_per_s": [18.7, 18.8150520415816, 19.05], "raw_mean_delta_percent": 8.21675755680824, "steps_mean_delta_percent": 4.8956011177602}, + {"concurrency": 2, "control_raw_tps": [74.55964771796953, 77.1430004517391], "proposal_raw_tps": [74.6559919940477, 77.0811406806786, 78.9], "control_steps_per_s": [28.122497998321393, 28.93488271324153], "proposal_steps_per_s": [28.2211658744255, 29.2687816368766, 29.600000000000005], "raw_mean_delta_percent": 1.35491390882609, "steps_mean_delta_percent": 1.75715093028919}, + {"concurrency": 4, "control_raw_tps": [120.7303004461975, 118.67889540432174], "proposal_raw_tps": [123.991379741799, 115.25, 123.18949531411437], "control_steps_per_s": [43.336510006445636, 43.30175913400928], "proposal_steps_per_s": [43.9031724550588, 41.8, 44.10364356241686], "raw_mean_delta_percent": 0.923685287957832, "steps_mean_delta_percent": -0.115874655007697}, + {"concurrency": 8, "control_raw_tps": [163.50687130131578, 171.3898781157694], "proposal_raw_tps": [164.267442443414, 173.45, 168.77003205133715], "control_steps_per_s": [59.384090681214076, 60.189597231174496], "proposal_steps_per_s": [61.393389175798, 61.0767548906789, 60.89743589745578], "raw_mean_delta_percent": 0.824602483468539, "steps_mean_delta_percent": 2.23407460582565} + ], + "prefill_prompt_tps_means": [ + {"context_tokens": 8192, "control": 2668.0, "proposal_head": 2670.33333333333, "delta_percent": 0.0874562718640837}, + {"context_tokens": 16384, "control": 2709.0, "proposal_head": 2707.0, "delta_percent": -0.0738279808047304}, + {"context_tokens": 65536, "control": 2782.5, "proposal_head": 2778.33333333333, "delta_percent": -0.149745432764292}, + {"context_tokens": 131072, "control": 2749.0, "proposal_head": 2739.33333333333, "delta_percent": -0.351643021704862} + ] +} diff --git a/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md b/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md new file mode 100644 index 00000000..f4a3d465 --- /dev/null +++ b/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md @@ -0,0 +1,122 @@ +# Native-MTP3 NVFP4 proposal-head comparison + +Status: **research-only**. These measurements support the proposal-head default +inside the research-only GLM-5.3 native-MTP3 mesh profile. They are not a general +model-quality or production-serving claim. + +## Conditions + +The measured system used four NVIDIA DGX Spark GB10 nodes in TP4/DCP4/PP1, +`local-inference-lab/GLM-5.3-Flash-NVFP4-Spark` revision +`df116c4fb16b1d37ae43d2cfd624de26ffbc832e`, native MTP depth three, FP8 KV, +24 GiB KV allocation per rank, SparkCache, and the profile's hybrid +SIRCL/RoCEnante transport. No DFlash model was loaded. + +The measured proposal-head image is +`sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74`. +Its image receipt has SHA-256 +`aa8f1bf3d892af7b0a8059626ec91a83e2dbc5a39c8d58c0fb82c614e88dbaba`; +the completed model-readiness and live-control receipts have SHA-256 +`fe786299a6459cd57dcf5362591b56d94569dd473b0d1fefb47ca374399983f3` +and `b0ebcf65be43998791f66da819ebaa98aa4b4323b9b76cc1e363d977750e3341`. +The image is layered on compute image +`sha256:f35ed3d1df1ee57f66ba571a491d6dd5575d1a2e619cbc63d065008894777ffe`, +which records CUDA 13.3.33, complete B12X revision +`b58f34eaf978277621efced6678e6713fd7122e4`, the uniform-speculation metadata +port from Local Inference Lab vLLM revision +`3512b066e7796128c0c380ccc558182960f2f0ea`, and dense wrapper changes from +revision `a8c796f3af74106b2d8d441e9ec54588936a5388`. The complete B12X tree also +contains MoE and other September changes, so these runs do not isolate a dense +kernel contribution. + +The changed variable was a separate runtime-NVFP4 proposal head with BF16 +activations. The target/verifier head kept its BF16 checkpoint representation. +The proposal head adds 85.08 MiB of persistent packed weight and scale storage +per rank while the retained BF16 target head remains allocated. It is not a net +85.08 MiB model-memory reduction. The shared-BF16-head control used the same +metadata+dense+B12X base without the separate proposal allocation. +Thus the proposal-head comparison preserves the verifier implementation. The +complete compute image still differs from the previous public image in CUDA, +metadata, dense, MoE, and B12X code; “BF16 verifier retained” is not a claim +that every target-side kernel is unchanged across those images. + +The benchmark used harness version 0.4.32, temperature 1.0, ignored EOS, a +20-second sustained-decode window, 8,192-token decode context, and concurrency +1, 2, 4, and 8. The proposal-head configuration has three repetitions and the +control has two. All compared 8K cells had +zero request errors, no warmup timeout, and no underfill flag. + +The benchmark JSON does not embed the image ID. The image, launch, and completed +four-rank readiness receipts bind the benchmark endpoint to the tested private +image. The compute-equivalent public derivative is +`ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86` +with image ID +`sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df`. +Its compute-equivalence record matches all vLLM, B12X, and SparkCache package +files and the selected environment to the tested image. Exact-image serving, +restart, and persistent-cache checks were not repeated on the public image ID. + +## Measurement + +Raw throughput is aggregate output tokens per second. Normalized throughput is +the harness's aggregate sequence steps per second, computed from drafted and +non-speculative request work. It is not a count of batched engine iterations. +Speculative acceptance can move raw tokens per second, so both metrics are +reported. The machine-readable values and receipt hashes are in +[`spark-mtp3-nvfp4-proposal-head-20260905.json`](spark-mtp3-nvfp4-proposal-head-20260905.json). + +Proposal-head receipt hashes: + +- `122857`: `05d846465153230f4312a59aefef12c499d81634855f3c84cb6757baf653c0e4` +- `123541`: `9664bc0410db423d3edc759106f10bc24a0fe1d15952102cdd74913ff96c2149` +- `124222`: `4213df0fc47ba53308ea7dc076e3e1c77e9097e5eaef95db4e09ca4cd404e434` + +Shared-BF16-head control hashes: + +- `114306`: `f7ae32cef4c9ea606477a28ff0465403e78f9519e2d771e4af0f13017b671677` +- `115629`: `7c3ff3210e805b8a801d4e8ea49152d0545a982707cc9010a174f77ff3d4e673` + +The previously published native-MTP3 `233405` receipt is retained as historical context +with SHA-256 `f0916f6b72cb8256225169b44c4f11e3ca764a5dd854977b8963686197b843fa`. +Its basename contains a stale DFlash7 label, but the public performance record +identifies the measured runtime as native MTP3. Its matrix and decode warmup +differ, and its JSON does not attest an image identity. It is not used to +calculate the proposal-head delta. + +## Result + +| 8K concurrency | Control raw tok/s | NVFP4-head raw tok/s | Raw change | Control steps/s | NVFP4-head steps/s | Normalized change | +|---:|---:|---:|---:|---:|---:|---:| +| 1 | 47.73 | 51.65 | **+8.22%** | 17.98 | 18.86 | **+4.90%** | +| 2 | 75.85 | 76.88 | +1.35% | 28.53 | 29.03 | +1.76% | +| 4 | 119.70 | 120.81 | +0.92% | 43.32 | 43.27 | -0.12% | +| 8 | 167.45 | 168.83 | +0.82% | 59.79 | 61.12 | +2.23% | + +The C1 repetitions show a small-batch benefit: approximately 8% +raw throughput and 5% normalized throughput. C2, C4, and C8 are mixed and do +not support a monotonic concurrency-wide speedup claim. + +Mean prefill scouts were unchanged within ±0.36%: 8K 2,668 versus 2,670.3 +prompt tok/s, 16K 2,709 versus 2,707, 64K 2,782.5 versus 2,778.3, and 128K +2,749 versus 2,739.3. The head change is decode-facing; these observations do +not establish a prefill improvement. + +## Conclusion + +The bounded evidence supports making runtime NVFP4/BF16 proposal-head execution +the default within this opt-in, research-only profile. It improves repeated C1 +decode while remaining approximately flat or mixed at the measured higher +concurrencies and prefill contexts. Relative to the compute-image control, the +verifier implementation and BF16 head are fixed; the proposal change can alter +speculative acceptance but not the target distribution under standard +rejection sampling. + +## Limitations + +The two control and three proposal-head repetitions are a small sample. They +cover only C1/C2/C4/C8 at 8K for +the direct comparison, not the full C1–C16 and 8K/32K/64K matrix. They do not +include isolated dense or MoE attribution, a general +accuracy evaluation, host reboot, long soak, failure containment, or unattended +high availability. A full C1–C16 and 8K/32K/64K matrix requires a separate +completed receipt before it can extend this record. diff --git a/recipes/glm53-spark-mtp3-managed-mesh-tp4.json b/recipes/glm53-spark-mtp3-managed-mesh-tp4.json index 1d60217f..94d0e43f 100644 --- a/recipes/glm53-spark-mtp3-managed-mesh-tp4.json +++ b/recipes/glm53-spark-mtp3-managed-mesh-tp4.json @@ -34,8 +34,10 @@ "operator_contract": "runtime/glm53-spark-mtp3-mesh/pins.json", "public_image_contract": "runtime/glm53-spark-mtp3-mesh/public-image.json", "image_receipt": "runtime/glm53-spark-mtp3-mesh/image-receipt.json", - "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:23f00af873ccc784cfb742b7be2a29c6d3c20ebec9741843c025320bb9c04685", - "image_id": "sha256:26273b8e358df139ae913610a5d43084ff0fd08aafe282ef633a3bc74afefe47", + "compute_contract": "runtime/glm53-spark-mtp3-mesh/compute/source-lock.json", + "compute_contract_sha256": "2a444f7c1ad4319f64afbffb16403491a4863934372cf818f18c510d58c0d00e", + "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86", + "image_id": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", "site_template": "runtime/glm53-spark-mtp3-mesh/site.example.json", "fabric_template": "runtime/glm53-spark-mtp3-mesh/fabric.example.json", "renderer": "runtime/glm53-spark-mtp3-mesh/profile.py", @@ -63,7 +65,23 @@ "linear_backend": "B12X", "kda_prefill_backend": "b12x", "cudagraph_capture_sizes": [4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64], - "speculation": {"method": "mtp", "num_speculative_tokens": 3, "draft_tensor_parallel_size": 4}, + "speculation": { + "method": "mtp", + "num_speculative_tokens": 3, + "draft_tensor_parallel_size": 4, + "proposal_head": { + "weight_quantization": "runtime_nvfp4", + "activation_dtype": "bfloat16", + "checkpoint_source": "target lm_head.weight", + "persistent_allocation_bytes_per_rank": 89210880, + "target_verifier_head": "retained BF16 checkpoint representation", + "environment": { + "VLLM_MTP_NVFP4_LM_HEAD": "1", + "VLLM_LM_HEAD_A16": "1", + "VLLM_MXFP8_LM_HEAD": "0" + } + } + }, "readiness_warmup_temperature": 1.0, "readiness_warmup_thinking": false }, @@ -82,6 +100,7 @@ "async_page_capture": true, "capture_slots_per_rank": 2, "capture_slot_bytes": 3221225472, + "namespace": "glm53-spark-df116c4f-mtp3-nvfp4-a16-b58f34ea-mesh4204fabc-tail-cow-v2", "identity_contract": "runtime/glm53-spark-mtp3-mesh/pins.json#/cache_identity" }, "transport": { @@ -100,10 +119,12 @@ }, "evidence": { "record": "performance/records/glm53-flash/spark-mtp3-managed-mesh-functional-20260905.md", - "status": "qualified", - "scope": "Bounded native correctness, managed lifecycle, model startup and persistent recall checks for the recorded image and conditions", + "proposal_head_performance_record": "performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md", + "status": "research-only", + "scope": "The proposal-head performance screen covers the recorded private image. Managed lifecycle and persistent-recall qualification remain bound to the previous published image.", "limitations": [ "This opt-in profile does not replace the recommended DFlash/SIRCL profile.", + "The published compute image has build/content equivalence to the tested private image but requires fresh native, startup, restart and persistent-cache receipts before those qualifications transfer.", "No host-reboot, prolonged-soak, unattended high-availability or general cache/model-quality qualification is claimed.", "Fault detection has a bounded observation window; partial streamed output can precede shutdown.", "Readiness warmup covers temperature-one sampling with thinking disabled, not all mixed-prefill/decode shapes." diff --git a/runtime/glm53-spark-mtp3-mesh/Dockerfile b/runtime/glm53-spark-mtp3-mesh/Dockerfile index 6a4982aa..5e97016b 100644 --- a/runtime/glm53-spark-mtp3-mesh/Dockerfile +++ b/runtime/glm53-spark-mtp3-mesh/Dockerfile @@ -5,6 +5,18 @@ ARG PARENT_IMAGE ARG PARENT_IMAGE_ID ARG BUNDLE_MANIFEST_SHA256 ARG SOURCE_RECEIPT_SHA256 +ARG COMPUTE_SOURCE_LOCK_SHA256 + +COPY compute/ /opt/sparkring-compute/ +ENV CUDA_HOME=/opt/cuda-13.3 CUDA_PATH=/opt/cuda-13.3 +ENV PATH="/opt/cuda-13.3/bin:${PATH}" +ENV LD_LIBRARY_PATH="/opt/cuda-13.3/lib:/opt/cuda-13.3/lib64:${LD_LIBRARY_PATH}" +ENV TRITON_PTXAS_PATH=/opt/cuda-13.3/bin/ptxas +ENV VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH=1 VLLM_B12X_DENSE_ACTIVATION_MODE=auto +ENV VLLM_MTP_NVFP4_LM_HEAD=1 VLLM_LM_HEAD_A16=1 VLLM_MXFP8_LM_HEAD=0 +RUN python3 /opt/sparkring-compute/apply_compute.py \ + --prepared /opt/sparkring-compute \ + --site-packages /usr/local/lib/python3.12/dist-packages COPY bundle/ /opt/spark-sircl/ COPY receipts/ /opt/sparkring/receipts/glm53-spark-mtp3-mesh/ @@ -27,4 +39,8 @@ LABEL org.opencontainers.image.title="SparkRing GLM-5.3 Spark MTP3 mesh runtime" org.sparkring.mesh.bundle-manifest-sha256="${BUNDLE_MANIFEST_SHA256}" \ org.sparkring.mesh.source-receipt-sha256="${SOURCE_RECEIPT_SHA256}" \ org.sparkring.mesh.default-speculation="mtp3" \ + org.sparkring.compute.source-lock-sha256="${COMPUTE_SOURCE_LOCK_SHA256}" \ + org.sparkring.b12x.composition="b58f34eaf978277621efced6678e6713fd7122e4" \ + org.sparkring.vllm.compute-source="${COMPUTE_SOURCE_LOCK_SHA256}" \ + org.sparkring.mesh.proposal-head="nvfp4-a16" \ org.sparkring.sircl.manifest-sha256="${BUNDLE_MANIFEST_SHA256}" diff --git a/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md b/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md index 0b7ce6ff..cc9296b9 100644 --- a/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md +++ b/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md @@ -1,14 +1,23 @@ -# ARM64 image with the MTP3 mesh bundle - -Status: **research-only**. This child image packages the transport -files used by the GLM-5.3 Spark native-MTP3 profile. It does not change the -parent image's vLLM, B12X model kernels, SparkCache, or NCCL. It replaces the -readiness warmup helper and sets `SPARKRING_WARMUP_TEMPERATURE=1`, so requests -issued before readiness use temperature one. This is an explicit startup -behavior override, not a claim that every parent entrypoint component is -unchanged. The managed quickstart requires this child and its verified -receipt. The parent-image rendering mode is a separate composition interface; -it does not meet the managed marker and temperature-one warmup contract. +# ARM64 image with native-MTP3 compute and mesh bundle + +Status: **research-only**. This child image packages the complete compute and +transport composition used by the GLM-5.3 Spark native-MTP3 profile. On top of +the pinned parent, it installs CUDA 13.3, the checksum-bound vLLM metadata and +proposal-head patch, complete B12X revision `b58f34ea`, the transport bundle, +the managed marker, and the readiness helper. The proposal head uses runtime +NVFP4 weights with BF16 activations; the target/verifier head retains its BF16 +checkpoint representation. SparkCache and patched NCCL remain inherited. + +The image sets `SPARKRING_WARMUP_TEMPERATURE=1`, so requests issued before +readiness use temperature one. The managed quickstart requires this child and +its verified receipt. The parent-image rendering mode is a separate composition +interface; it does not meet the compute, managed marker, or temperature-one +warmup contract. + +For the pinned GLM vocabulary and TP4 geometry, the proposal head adds about +85.08 MiB of packed NVFP4 values and scales per rank. It does not replace the +retained BF16 target/verifier head, so this is an additional persistent +allocation rather than a net model-memory reduction. The image contains no model weights. It does not provision NIC rules, select network interfaces, install host services, or start a model during construction. @@ -21,12 +30,20 @@ to the tested Linux/ARM64 image. Pull before using its local image ID: ```bash set -euo pipefail -mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:23f00af873ccc784cfb742b7be2a29c6d3c20ebec9741843c025320bb9c04685' -mtp_image_id='sha256:26273b8e358df139ae913610a5d43084ff0fd08aafe282ef633a3bc74afefe47' +mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86' +mtp_image_id='sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df' docker pull "$mtp_image" test "$(docker image inspect "$mtp_image" --format '{{.Id}}')" = "$mtp_image_id" ``` +The immutable reference is also published as tag +`glm53-spark-mtp3-nvfp4-a16-2a444f7c`; use the digest above for deployment. +The [compute-image equivalence record](compute-image-equivalence.json) verifies +that all 4,891 vLLM, 385 B12X, and 150 SparkCache package files and the selected +environment match tested private image `sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74`. +This is build/content equivalence, not a fresh serving, restart, or persistent- +cache qualification of the public image ID. + Use the repository's [content receipt](image-receipt.json) for the renderer, installer, and native qualification runner. This default deployment requires no compiler or local image rebuild. The source reproduction commands below @@ -37,6 +54,8 @@ are optional; skip to **Use and export** when using the published image. | Object | Identity or location | |---|---| | Parent runtime | `operator_image` in `../glm53-flash-jj-r8-gb10/pins.json` | +| Compute source lock and vLLM patch | [`compute/source-lock.json`](compute/source-lock.json), [`compute/vllm-e02-to-compute.patch`](compute/vllm-e02-to-compute.patch) | +| Prepared CUDA 13.3 and B12X source payload | output of [`compute/prepare_compute_source.py`](compute/prepare_compute_source.py) | | Target, speculation, transport, and marker pins | `pins.json` in this directory | | Embedded transport bundle | `/opt/spark-sircl` | | Compiled RDMA transmit marker | `/opt/sparkring/bin/mlx5-rdma-tx-marker` | @@ -61,9 +80,22 @@ Use a Linux/ARM64 Docker host. Check available disk and RAM before pulling the parent image. Do not remove resident model files or stop serving containers to make room for this build. -From the repository root, first pull the pinned parent and compose its SIRCL -bundle. Do not use the already-composed published child as the parent input. -Use absent output paths and an unused temporary container name: +From the repository root, first prepare the checksum-bound compute source. This +step downloads the pinned public CUDA component archives and Local Inference +Lab B12X source before the network-disabled image build. The output and optional +download cache are local build inputs and must use absent or empty paths: + +```bash +python3 runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py \ + --output /var/tmp/mtp3-compute-source \ + --cache /var/tmp/mtp3-compute-downloads +``` + +The preparer verifies every archive checksum, B12X tree, vLLM patch hash, and +package file and writes `prepared-manifest.json`. Then pull the pinned parent +and compose its SIRCL bundle. Do not use the already-composed published child +as the parent input. Use absent output paths and an unused temporary container +name: ```bash mtp_parent='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:0d4029b3b7023cf32c37ac20279469c9a2ee16a057f25aae3bcfee9ee5fb660f' @@ -83,13 +115,14 @@ Then prepare the content-verified build context: ```bash python3 runtime/glm53-spark-mtp3-mesh/build_image.py prepare \ --bundle /var/tmp/mtp3-mesh-bundle \ + --compute-context /var/tmp/mtp3-compute-source \ --context /var/tmp/mtp3-mesh-image-context ``` The context path must not exist. Preparation copies only manifest-listed -transport files, source-pinned marker code, verification code, pins, and the -RoCEnante license and provenance. It rejects unexpected bundle files and writes -a content manifest for every construction input. +compute and transport files, source-pinned marker code, verification code, +pins, and the RoCEnante license and provenance. It rejects unexpected bundle +or compute files and writes a content manifest for every construction input. Build and verify without loading a model: @@ -109,8 +142,9 @@ image to a registry. The local tag is a convenience; use the receipt's full Image construction has networking disabled. Verification runs with no host device mounts, no Linux capabilities, no network, a read-only root filesystem, two CPUs, and a 2 GiB memory limit. It checks the complete parent layer prefix, -package source and native-library hashes, bundle hashes, Python syntax, readiness -warmup helper hash and temperature environment, lazy +package source and native-library hashes, CUDA 13.3 components, the complete +B12X `b58f34ea` package, vLLM input/output hashes and proposal-head environment, +bundle hashes, Python syntax, readiness warmup helper hash and temperature, lazy RoCEnante import, and marker linkage. CUDA must remain uninitialized. ## Use and export diff --git a/runtime/glm53-spark-mtp3-mesh/README.md b/runtime/glm53-spark-mtp3-mesh/README.md index 74f7b88e..88c1efb2 100644 --- a/runtime/glm53-spark-mtp3-mesh/README.md +++ b/runtime/glm53-spark-mtp3-mesh/README.md @@ -12,10 +12,16 @@ recall restoration for its exact image. Broader cache/workload coverage and failure containment remain unqualified. This profile serves the `GLM-5.3-Flash-NVFP4-Spark` checkpoint with its built-in -multi-token predictor at depth three. It combines graph-native SIRCL, -dual-rail fused SIRCL, and a modified RoCEnante all-reduce over a four-node -physical ring. Opposite ranks communicate through hardware forwarding in an -intermediate ConnectX-7. No external draft checkpoint is required. +multi-token predictor at depth three. The predictor uses a separate runtime- +NVFP4 proposal head with BF16 activations; the target/verifier head retains its +BF16 checkpoint representation. It combines graph-native SIRCL, dual-rail +fused SIRCL, and a modified RoCEnante all-reduce over a four-node physical +ring. Opposite ranks communicate through hardware forwarding in an intermediate +ConnectX-7. No external draft checkpoint or DFlash model is required. + +At TP4, the proposal head adds approximately 85.08 MiB of packed NVFP4 values +and scales per rank. The BF16 target/verifier head remains allocated. The +proposal-head figure is an added allocation, not a net memory reduction. Follow the [operator quickstart](../../docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md). It starts from a public checkout and image/model artifacts, explains the @@ -24,8 +30,9 @@ installation commands. No private experiment checkout or existing cache is required. The [managed operations guide](MANAGED_MESH.md) creates the shared authentication inputs and provides model-start, readiness, stop, and recovery commands. Keep private site files and the health key outside this repository. -The [measurement record](../../performance/records/glm53-flash/spark-mtp3-mesh-20260905.md) -contains the bounded throughput observations and their limitations. +The [proposal-head comparison](../../performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md) +contains the bounded throughput observations for this head configuration and +their limitations. ## Operator benchmark observations @@ -34,20 +41,33 @@ collects the completed tests, three-pass prefill measurements, and remaining work. Use that report to avoid repeating checks already covered by receipts. Status: **research-only** measurements, not general performance guarantees. -The [throughput record](../../performance/records/glm53-flash/spark-mtp3-mesh-20260905.md) -identifies the measured source configuration and its differences from the -packaged image. C denotes concurrent requests; decode values are aggregate -output tokens per second across those requests. - -| Context | C1 | C2 | C4 | C8 | C12 | C16 | -|---:|---:|---:|---:|---:|---:|---:| -| 8K | 48.2 | 75.8 | 112.2 | 168.8 | 193.4 | 231.3 | -| 32K | 49.9 | 76.8 | 119.0 | 164.6 | 197.3 | 222.7 | -| 64K | 43.0 | 76.4 | 119.0 | 165.8 | 192.3 | 220.9 | - -Concurrency-one prefill scouts measured **2,703–2,787 prompt tokens/s** over -8K–128K contexts, with one observation per context. They are not a repeated -cold-cache benchmark. +The [proposal-head comparison](../../performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md) +reports three proposal-head repetitions and two controls at 8K. C denotes +concurrent requests; decode values are +mean aggregate output tokens per second across those requests. + +| Context | C1 | C2 | C4 | C8 | +|---:|---:|---:|---:|---:| +| 8K | 51.6 | 76.9 | 120.8 | 168.8 | + +Relative to the same metadata+dense+B12X composition with a shared BF16 +proposal head, C1 improved 8.22% in raw output throughput and 4.90% in +acceptance-normalized sequence steps/s. C2/C4/C8 results were mixed. Repeated +prefill means moved by no more than 0.36% over 8K–128K contexts, so no prefill +gain is claimed. The earlier +[broader matrix](../../performance/records/glm53-flash/spark-mtp3-mesh-20260905.md) +belongs to the previous image configuration and remains historical context. +The head-specific control preserves the verifier implementation; the complete +CUDA 13.3/B12X `b58f34ea` image changes other target computation relative to +the previous public image, so cross-image gains cannot be assigned only to the +proposal head or dense kernels. + +The published image's +[compute-equivalence record](compute-image-equivalence.json) matches every +vLLM, B12X, and SparkCache package file and selected environment entry to the +tested private image. That supports applying the recorded compute result to the +published bytes. Native transport, model startup, restart, and persistent-cache +checks remain image-ID-specific and have not been repeated on the public image. The separate [Estonia accuracy record](../../performance/records/glm53-flash/spark-mtp3-country-recall-20260905.md) reports **30/30 correct** at C8 on one repeated 133,208-token prompt, no @@ -64,16 +84,19 @@ passed **4/4** exact-value, revision, and cross-reference checks, reaching | Input | Contract | |---|---| | Model, MTP depth, graph shapes, mesh bundle, marker identity, cache identity | [`pins.json`](pins.json) | -| Linux/ARM64 image, vLLM, B12X kernels, SparkCache, native SIRCL | [`../glm53-flash-jj-r8-gb10/pins.json`](../glm53-flash-jj-r8-gb10/pins.json) | +| Linux/ARM64 parent image, SparkCache, and native SIRCL | [`../glm53-flash-jj-r8-gb10/pins.json`](../glm53-flash-jj-r8-gb10/pins.json) | +| CUDA 13.3, GLM metadata port, complete B12X `b58f34ea`, and runtime-NVFP4/BF16 proposal head | [`pins.json`](pins.json), [`IMAGE_BUILD.md`](IMAGE_BUILD.md) | | Topology and rank-local filesystem inputs | [`site.example.json`](site.example.json), [`fabric.example.json`](fabric.example.json) | | Source-bound collective dispatch and health checks | [`glm53_rocenante_overlay`](../../spark_transport/experiments/glm53_rocenante_overlay/README.md) | | Hardware-forwarding plan and native source marker | [`cx7_hairpin_diagonal`](../../spark_transport/experiments/cx7_hairpin_diagonal/README.md) | | Modified RoCEnante communication package | [`third_party/b12x_roce`](../../third_party/b12x_roce/README.md) | -The model runtime and kernels come from the pinned parent image. The managed -profile requires the [published child image](IMAGE_BUILD.md), which adds -the verified transport bundle, managed source marker, and temperature-one -readiness helper. Pull the immutable reference in [public-image.json](public-image.json) +The managed profile requires the [published child image](IMAGE_BUILD.md). It +retains the parent runtime while adding CUDA 13.3, the uniform native-MTP3 +metadata port, complete B12X revision `b58f34ea`, the runtime-NVFP4/BF16 +proposal head, the verified transport bundle, the managed source marker, and +the temperature-one readiness helper. The verifier remains BF16. Pull the +immutable reference in [public-image.json](public-image.json) and use [image-receipt.json](image-receipt.json) for rendering and installation. Local source reproduction is optional; distribute identical verified bytes to all ranks. The canonical transport bundle remains mounted read-only. @@ -140,7 +163,8 @@ serving lifecycle. Use authenticated managed readiness for serving. Native MTP uses the target checkpoint as the draft identity. The profile sets SparkCache's `draft_policy=separate` because that describes the registered state layout; it does not request an external model. A dedicated namespace -prevents restoration of entries tagged for an external DFlash checkpoint. +includes `mtp3-nvfp4-a16-b58f34ea` so the new compute and proposal-head +composition cannot restore shared-BF16-head or external-DFlash entries. Do not relabel those entries to avoid cache misses. Persistent restore under the native-MTP identity requires its own qualification. diff --git a/runtime/glm53-spark-mtp3-mesh/build_image.py b/runtime/glm53-spark-mtp3-mesh/build_image.py index 8e48309f..65991aee 100644 --- a/runtime/glm53-spark-mtp3-mesh/build_image.py +++ b/runtime/glm53-spark-mtp3-mesh/build_image.py @@ -67,11 +67,15 @@ def verify_bundle(bundle: Path, expected: str) -> list[dict]: return records -def prepare(bundle: Path, context: Path) -> dict: +def prepare(bundle: Path, context: Path, compute: Path | None = None) -> dict: """Copy only content-verified inputs into a directory that does not exist.""" if context.exists(): raise ValueError(f"Build context already exists: {context}") + if compute is None or not compute.is_dir() or compute.is_symlink(): + raise ValueError("A prepared compute source directory is required; see compute/README.md") profile = read_json(HERE / "pins.json") + if sha256(compute / "source-lock.json") != profile["compute"]["source_lock_sha256"]: + raise ValueError("Prepared compute source lock differs from the mesh profile pin") base_path = (HERE / profile["image_pins"]).resolve() base = read_json(base_path) records = verify_bundle(bundle, profile["canonical_bundle_manifest_sha256"]) @@ -95,6 +99,11 @@ def prepare(bundle: Path, context: Path) -> dict: f"bundle/{MANIFEST}": bundle / MANIFEST, } files.update({f"bundle/{record['path']}": bundle / record["path"] for record in records}) + for source in compute.rglob('*'): + if source.is_symlink(): + raise ValueError(f"Compute inputs cannot contain symlinks: {source}") + if source.is_file(): + files[f"compute/{source.relative_to(compute).as_posix()}"] = source for relative, source in files.items(): destination = context / relative destination.parent.mkdir(parents=True, exist_ok=True) @@ -109,7 +118,7 @@ def prepare(bundle: Path, context: Path) -> dict: "helper_sha256": sha256(context / "warmup_dflash.py"), "temperature_environment": "SPARKRING_WARMUP_TEMPERATURE", "default_temperature": 1.0}, - "scope": "Embedded transport bundle, compiled host marker, and readiness warmup helper with temperature one; target weights are not image contents.", + "scope": "Compute source and CUDA components, embedded transport bundle, compiled host marker, and temperature-one readiness warmup; target weights are not image contents.", } write_json(context / "receipts/source-receipt.json", receipt) return receipt @@ -162,6 +171,7 @@ def build(context: Path, image: str, receipt_path: Path, engine: str, pull: bool "PARENT_IMAGE_ID": parent["image_id"], "BUNDLE_MANIFEST_SHA256": source["bundle_manifest_sha256"], "SOURCE_RECEIPT_SHA256": source_sha, + "COMPUTE_SOURCE_LOCK_SHA256": source["files"]["compute/source-lock.json"], }.items(): argv.extend(["--build-arg", f"{name}={value}"]) argv.extend(["--file", str(context / "Dockerfile"), "--tag", image, str(context)]) @@ -180,6 +190,8 @@ def main() -> int: prepare_parser = sub.add_parser("prepare", help="OFFLINE: construct a content-verified build directory") prepare_parser.add_argument("--bundle", type=Path, required=True) prepare_parser.add_argument("--context", type=Path, required=True) + prepare_parser.add_argument("--compute-context", type=Path, required=True, + help="Prepared, source-pinned compute directory from compute/prepare_compute_source.py") build_parser = sub.add_parser("build", help="MUTATES HOST: build and CPU-check an image; no GPU or fabric access") build_parser.add_argument("--context", type=Path, required=True) build_parser.add_argument("--image", required=True) @@ -188,7 +200,7 @@ def main() -> int: build_parser.add_argument("--pull-parent", action="store_true") args = parser.parse_args() if args.command == "prepare": - result = prepare(args.bundle.resolve(), args.context.resolve()) + result = prepare(args.bundle.resolve(), args.context.resolve(), args.compute_context.resolve()) print(json.dumps({"context": str(args.context.resolve()), "bundle_manifest_sha256": result["bundle_manifest_sha256"]}, indent=2)) else: build(args.context.resolve(), args.image, args.receipt.resolve(), args.engine, args.pull_parent) diff --git a/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json b/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json new file mode 100644 index 00000000..3d2de485 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json @@ -0,0 +1,26 @@ +{ + "schema": "sparkring-compute-image-equivalence/v1", + "checks_passed": true, + "tested_serving_image": "sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74", + "published_image": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", + "package_files": { + "b12x": 385, + "sparkcache": 150, + "vllm": 4891 + }, + "package_files_identical": true, + "selected_environment_identical": true, + "tested_snapshot_sha256": "54a975edcb68df32b6520f7070b21e0b88e280f847d5039c81bb1f2f99f7cc51", + "published_snapshot_sha256": "54a975edcb68df32b6520f7070b21e0b88e280f847d5039c81bb1f2f99f7cc51", + "environment": { + "CUDA_HOME": "/opt/cuda-13.3", + "SPARKRING_WARMUP_TEMPERATURE": "1", + "TRITON_PTXAS_PATH": "/opt/cuda-13.3/bin/ptxas", + "VLLM_B12X_DENSE_ACTIVATION_MODE": "auto", + "VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH": "1", + "VLLM_LM_HEAD_A16": "1", + "VLLM_MTP_NVFP4_LM_HEAD": "1", + "VLLM_MXFP8_LM_HEAD": "0" + }, + "scope": "Installed package files excluding Python bytecode and selected environment. Public image construction and native-marker verification passed. Full-model serving was measured on the tested serving image; no claim of a separate four-rank serving run on the published image." +} diff --git a/runtime/glm53-spark-mtp3-mesh/compute/README.md b/runtime/glm53-spark-mtp3-mesh/compute/README.md new file mode 100644 index 00000000..ba35aa58 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/compute/README.md @@ -0,0 +1,46 @@ +# GLM-5.3 compute source composition + +Status: **implemented**. These files reproduce the compute source used by the +GLM-5.3 NVFP4-Spark native-MTP3 mesh image. Hardware qualification belongs to +the profile's validation records, not to this source-preparation tooling. + +`source-lock.json` is the authoritative input. It binds: + +- the vLLM source revision already present in the parent image; +- a reviewable vLLM patch, a byte-exact 14-file replacement archive, and every + base and resulting file hash; +- B12X revision `b58f34eaf978277621efced6678e6713fd7122e4`, its Git tree, + source archive, and all 385 installed package-file hashes; +- seven NVIDIA CUDA 13.3 SBSA redistributable archives; and +- the five environment settings that select metadata reuse, dense-kernel + policy, and the NVFP4 native-MTP proposal head. + +The proposal head receives a separate tensor-parallel copy of the checkpoint's +unquantized `lm_head.weight`, converts that copy to NVFP4 during loading, and +uses BF16 activations. `VLLM_MXFP8_LM_HEAD=0` leaves the target/verifier head +unchanged. Rejection sampling therefore retains the target model's sampling +contract, while proposal-head quantization can change acceptance length. + +The image builder calls `prepare_compute_source.prepare(destination, cache)` +while network access is available. The prepared directory contains the pinned +B12X source and CUDA archives. Docker copies that directory into the build and +runs `apply_compute.py` with network access disabled. The installer verifies +the parent hashes before extracting the replacement archive; this preserves the +mixed line endings of the tested source without requiring Git in the image. +`verify_compute.py` +requires exact installed hashes and rejects missing or partial source maps. + +The B12X source is obtained from +[`local-inference-lab/b12x`](https://github.com/local-inference-lab/b12x) and is +licensed under Apache License 2.0. The downloaded source archive includes its +`LICENSE` file. The vLLM patch derives from +[`local-inference-lab/vllm`](https://github.com/local-inference-lab/vllm) +revision `3512b066e7796128c0c380ccc558182960f2f0ea`, with dense-kernel integration +from revision `a8c796f3af74106b2d8d441e9ec54588936a5388`; vLLM is licensed under +Apache License 2.0. + +B12X source archives use LF endings. The tested ARM64 image was assembled from +a Windows checkout and contains CRLF bytes for Python and C source files. The +preparation step performs that deterministic byte conversion so the public +image can be compared exactly with the tested image. Markdown and compressed +profile data retain the archive bytes. diff --git a/runtime/glm53-spark-mtp3-mesh/compute/apply_compute.py b/runtime/glm53-spark-mtp3-mesh/compute/apply_compute.py new file mode 100644 index 00000000..95311dad --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/compute/apply_compute.py @@ -0,0 +1,167 @@ +"""Install the prepared CUDA, B12X, and vLLM compute composition offline.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import subprocess +import tarfile +import tempfile +from pathlib import Path + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _map_sha256(files: dict[str, str]) -> str: + payload = json.dumps(files, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(payload).hexdigest() + + +def _load(prepared: Path) -> tuple[dict, dict]: + lock_path = prepared / "source-lock.json" + lock = json.loads(lock_path.read_text()) + manifest = json.loads((prepared / "prepared-manifest.json").read_text()) + if hashlib.sha256(lock_path.read_bytes()).hexdigest() != manifest[ + "source_lock_sha256" + ]: + raise ValueError("prepared source lock hash mismatch") + return lock, manifest + + +def _install_cuda(prepared: Path, lock: dict, destination: Path) -> None: + if destination.exists(): + raise ValueError(f"CUDA destination already exists: {destination}") + destination.mkdir(parents=True) + for relative, expected in lock["cuda"]["components"].items(): + archive = prepared / "cuda-archives" / Path(relative).name + if _sha256(archive) != expected: + raise ValueError(f"CUDA archive hash mismatch: {archive.name}") + with tempfile.TemporaryDirectory(prefix="sparkring-cuda-") as temporary: + unpack = Path(temporary) + with tarfile.open(archive) as source: + source.extractall(unpack, filter="data") + entries = list(unpack.iterdir()) + if len(entries) != 1 or not entries[0].is_dir(): + raise ValueError(f"invalid CUDA archive root: {archive.name}") + shutil.copytree(entries[0], destination, dirs_exist_ok=True, symlinks=True) + (destination / "sparkring-component-manifest.json").write_text( + json.dumps(lock["cuda"]["components"], indent=2, sort_keys=True) + "\n" + ) + + +def _install_vllm(prepared: Path, site: Path, lock: dict) -> dict[str, str]: + entries = lock["vllm"]["files"] + patch = prepared / lock["vllm"]["patch"] + if _sha256(patch) != lock["vllm"]["patch_sha256"]: + raise ValueError("vLLM patch hash mismatch") + archive = prepared / lock["vllm"]["replacement_archive"] + if _sha256(archive) != lock["vllm"]["replacement_archive_sha256"]: + raise ValueError("vLLM replacement archive hash mismatch") + with tempfile.TemporaryDirectory(prefix="sparkring-vllm-") as temporary: + work = Path(temporary) + for relative, base_hash, _ in entries: + installed = site / relative + actual = _sha256(installed) + if actual != base_hash: + raise ValueError(f"vLLM base hash mismatch for {relative}: {actual}") + staged = work / relative + staged.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(installed, staged) + with tarfile.open(archive) as source: + names = set(source.getnames()) + expected_names = {entry[0] for entry in entries} + if names != expected_names: + raise ValueError("vLLM replacement archive has an unexpected file set") + source.extractall(work, filter="data") + result: dict[str, str] = {} + for relative, _, expected in entries: + staged = work / relative + actual = _sha256(staged) + if actual != expected: + raise ValueError(f"vLLM result hash mismatch for {relative}: {actual}") + shutil.copyfile(staged, site / relative) + result[relative] = actual + return result + + +def _package_map(root: Path) -> dict[str, str]: + package = root / "b12x" + files = sorted( + path + for path in package.rglob("*") + if path.is_file() + and "__pycache__" not in path.parts + and path.suffix != ".pyc" + ) + if not files: + raise ValueError(f"no installed B12X package files under {package}") + return {path.relative_to(root).as_posix(): _sha256(path) for path in files} + + +def apply( + prepared: Path, + site_packages: Path, + receipt: Path, + cuda_destination: Path, +) -> Path: + prepared = prepared.resolve() + site_packages = site_packages.resolve() + lock, manifest = _load(prepared) + _install_cuda(prepared, lock, cuda_destination) + vllm_files = _install_vllm(prepared, site_packages, lock) + subprocess.run( + [ + "python3", + "-m", + "pip", + "install", + "--no-deps", + "--no-build-isolation", + "--force-reinstall", + str(prepared / "b12x-source"), + ], + check=True, + ) + b12x_files = _package_map(site_packages) + if b12x_files != manifest["b12x_files"]: + raise ValueError("installed B12X package differs from prepared source") + if _map_sha256(b12x_files) != lock["b12x"]["package_files_sha256"]: + raise ValueError("installed B12X package differs from source-lock.json") + output = { + "schema": "sparkring-glm53-compute-installed/v1", + "source_lock_sha256": manifest["source_lock_sha256"], + "vllm_revision": lock["vllm"]["base_revision"], + "vllm_overrides": vllm_files, + "b12x_revision": lock["b12x"]["revision"], + "b12x_tree": lock["b12x"]["tree"], + "b12x_files": b12x_files, + "b12x_package_files_sha256": lock["b12x"]["package_files_sha256"], + "cuda_components": lock["cuda"]["components"], + "environment": lock["environment"], + "target_head_quantization": False, + } + receipt.parent.mkdir(parents=True, exist_ok=True) + receipt.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n") + return receipt + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--prepared", type=Path, required=True) + parser.add_argument("--site-packages", type=Path, required=True) + parser.add_argument( + "--receipt", + type=Path, + default=Path("/opt/sparkring/receipts/glm53-compute-installed.json"), + ) + parser.add_argument("--cuda-destination", type=Path, default=Path("/opt/cuda-13.3")) + args = parser.parse_args() + print(apply(args.prepared, args.site_packages, args.receipt, args.cuda_destination)) + + +if __name__ == "__main__": + main() diff --git a/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py b/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py new file mode 100644 index 00000000..46eb33f8 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py @@ -0,0 +1,155 @@ +"""Prepare the network-fetched compute payload for an offline image build.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import tarfile +import tempfile +import urllib.request +from pathlib import Path + +HERE = Path(__file__).resolve().parent +LOCK = json.loads((HERE / "source-lock.json").read_text()) + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _map_sha256(files: dict[str, str]) -> str: + payload = json.dumps(files, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(payload).hexdigest() + + +def _download(url: str, expected: str, destination: Path) -> None: + if destination.is_file() and _sha256(destination) == expected: + return + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=destination.parent, delete=False) as stream: + temporary = Path(stream.name) + try: + urllib.request.urlretrieve(url, temporary) + actual = _sha256(temporary) + if actual != expected: + raise ValueError(f"download hash mismatch for {url}: {actual}") + temporary.replace(destination) + finally: + temporary.unlink(missing_ok=True) + + +def _extract_single_root(archive: Path, destination: Path) -> Path: + destination.mkdir(parents=True) + with tarfile.open(archive) as source: + source.extractall(destination, filter="data") + entries = list(destination.iterdir()) + if len(entries) != 1 or not entries[0].is_dir(): + raise ValueError(f"expected one source root in {archive.name}") + return entries[0] + + +def _package_map(root: Path, package: str) -> dict[str, str]: + package_root = root / package + files = sorted( + path + for path in package_root.rglob("*") + if path.is_file() + and "__pycache__" not in path.parts + and path.suffix != ".pyc" + ) + if not files: + raise ValueError(f"no package files found under {package_root}") + return { + path.relative_to(root).as_posix(): _sha256(path) + for path in files + } + + +def _normalize_b12x_bytes(root: Path) -> None: + suffixes = set(LOCK["b12x"]["normalized_text_suffixes"]) + for path in sorted((root / "b12x").rglob("*")): + if path.is_file() and path.suffix in suffixes: + data = path.read_bytes().replace(b"\r\n", b"\n") + path.write_bytes(data.replace(b"\n", b"\r\n")) + + +def prepare(destination: Path, cache: Path | None = None) -> Path: + """Create a complete, checksum-bound context for a network-disabled build.""" + destination = destination.resolve() + if destination.exists() and any(destination.iterdir()): + raise ValueError(f"compute destination is not empty: {destination}") + destination.mkdir(parents=True, exist_ok=True) + cache = (cache or destination.parent / ".compute-downloads").resolve() + cache.mkdir(parents=True, exist_ok=True) + + for name in ( + "source-lock.json", + "vllm-e02-to-compute.patch", + "vllm-compute-files.tar.gz", + "apply_compute.py", + "verify_compute.py", + ): + shutil.copy2(HERE / name, destination / name) + + lock_hash = _sha256(destination / "source-lock.json") + patch = destination / LOCK["vllm"]["patch"] + if _sha256(patch) != LOCK["vllm"]["patch_sha256"]: + raise ValueError("vLLM patch does not match source-lock.json") + replacement = destination / LOCK["vllm"]["replacement_archive"] + if _sha256(replacement) != LOCK["vllm"]["replacement_archive_sha256"]: + raise ValueError("vLLM replacement archive does not match source-lock.json") + + b12x = LOCK["b12x"] + b12x_archive = cache / "b12x.tar.gz" + _download(b12x["archive_url"], b12x["archive_sha256"], b12x_archive) + unpack = destination / ".b12x-unpack" + source_root = _extract_single_root(b12x_archive, unpack) + b12x_destination = destination / "b12x-source" + shutil.move(str(source_root), b12x_destination) + shutil.rmtree(unpack) + _normalize_b12x_bytes(b12x_destination) + b12x_files = _package_map(b12x_destination, "b12x") + if _map_sha256(b12x_files) != b12x["package_files_sha256"]: + raise ValueError("normalized B12X package does not match source-lock.json") + + cuda_archives: dict[str, str] = {} + cuda_dir = destination / "cuda-archives" + cuda_dir.mkdir() + for relative, expected in LOCK["cuda"]["components"].items(): + archive = cache / Path(relative).name + _download(LOCK["cuda"]["base_url"] + relative, expected, archive) + target = cuda_dir / archive.name + shutil.copy2(archive, target) + cuda_archives[f"cuda-archives/{target.name}"] = expected + + prepared = { + "schema": "sparkring-glm53-compute-prepared/v1", + "source_lock_sha256": lock_hash, + "vllm_patch_sha256": LOCK["vllm"]["patch_sha256"], + "vllm_replacement_archive_sha256": LOCK["vllm"][ + "replacement_archive_sha256" + ], + "b12x_revision": b12x["revision"], + "b12x_tree": b12x["tree"], + "b12x_archive_sha256": b12x["archive_sha256"], + "b12x_package_files_sha256": b12x["package_files_sha256"], + "b12x_files": b12x_files, + "cuda_archives": cuda_archives, + } + manifest = destination / "prepared-manifest.json" + manifest.write_text(json.dumps(prepared, indent=2, sort_keys=True) + "\n") + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cache", type=Path) + args = parser.parse_args() + print(prepare(args.output, args.cache)) + + +if __name__ == "__main__": + main() diff --git a/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json b/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json new file mode 100644 index 00000000..7ebb33a4 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json @@ -0,0 +1,59 @@ +{ + "schema": "sparkring-glm53-compute-source/v1", + "status": "research-only", + "vllm": { + "base_revision": "e02b174693e13859de61811b5e8cd13d5308e259", + "donor_revision": "3512b066e7796128c0c380ccc558182960f2f0ea", + "dense_donor_revision": "a8c796f3af74106b2d8d441e9ec54588936a5388", + "patch": "vllm-e02-to-compute.patch", + "patch_sha256": "a41df4b7a6f2ab4c73157349cb2c66e9d89ccff05af124eb66783b849ae98e8a", + "replacement_archive": "vllm-compute-files.tar.gz", + "replacement_archive_sha256": "cefc8e7924404280e8ae024392d326fac3262a77d260af514d981921fdc93f90", + "files": [ + ["vllm/envs.py", "22069819122e630dd5131627c6bb8752820be2b66178d7f445eeb6ce93e03d32", "1fbac28f1a763f9d27845a8e09d59f71ba1740b4491a74d0097a7e9d63763804"], + ["vllm/model_executor/kernels/linear/mxfp8/b12x.py", "7abc42bccf03114e880871fa2ffd67d11466483b2ea636d466e762c17417f3d9", "1b4448f7dbaadcf9ef93b59a6aa54a016653a1019ffa31605722d828ea3d80af"], + ["vllm/model_executor/kernels/linear/nvfp4/b12x.py", "4c59ca2067e78a731598a47d146b9e8ddb1b8270f54f6ab19b72c30561f9d53d", "7a7036e6d0254a0ee6e3265b18685128fc42cb1b896efffdf27c9de038d69fc4"], + ["vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py", "efb07f928463fcf12a3c079f4da2ffa330d697a98a6afca3bea94a0a21825f58", "4a83c1eb9ea9c47c34e8a87b07f087a4cf1384428e74d800e0bdbd667abbb63e"], + ["vllm/model_executor/layers/quantization/online/nvfp4.py", "52ce69a32611d8eb00fa2a27cb9b2ec46b8836f34f0a1f8b4834990c3ea228a9", "8f85d00e6ef0effa873605e506c5c678228cba1459c2245a0770e2a3d0f74450"], + ["vllm/model_executor/layers/vocab_parallel_embedding.py", "e86d37bc50b5171e2bc02a8e6428feed34309807f63d20bead6bc5db4ac7fddf", "0f958f55912dd1f6bf1f902110404018263afd9c0aa34ebb66236780c73b9243"], + ["vllm/model_executor/models/deepseek_mtp.py", "e9270724c39a0152dc0a66b94622ebddd384c592534cbbf38d2f43c0ba1592d0", "ed5b247d014207e81d7d2ddf88e62e82afba70427798428b1b770a4f687ee869"], + ["vllm/models/glm5next/model_state.py", "8c66d55da1bcad63f703b1a5463b6969dfa1d3308d3d14abfea778f6c5e34b7f", "4f884f713335d55fafd729fa4cc9bcd88389ba005323eb79a3f9b02cf5662f36"], + ["vllm/models/glm5next/nvidia/mtp.py", "cbff653af56b3589a1ff3d52b5fea660e718cbdb1b278c4fa5dcd8eac0908ee9", "71e88e3a25d829d52d90a394f7306e29759d3951fb8f912d1279fa0f30496099"], + ["vllm/utils/b12x.py", "f610dc19b4dc10d27361b075ff7e8a97a63244f09f770dfc2ff04aeceed4dce1", "63e7fef8c75f5cd01678b338b9154fc1204b875794d0356904c461f4e9db3076"], + ["vllm/v1/attention/backends/gdn_attn.py", "7c325bbcb612aacd2411b3305ee6d068b013f8b6c4445eaea59059ac45881a3d", "fd75fb72efeb762ec558d332364e889df44a9ea483026bd5ded8f77f42be9b7f"], + ["vllm/v1/worker/gpu/attn_utils.py", "012399dde8910deec550df260c132338d20e20543acc7433ace60fec1229814b", "80108d185a996e52466f2348d6157398248acfd05b735e4a5792949034b70b9d"], + ["vllm/v1/worker/gpu/model_states/mamba_hybrid.py", "f2bc9ef85896df2508557bfbbcf2c682f82414595e8a562efdee6ed3dd0515c2", "b34cb130e233f4d322390acf7fec01a3a368090c7f79da1fa7770c622a2f0dde"], + ["vllm/v1/worker/mamba_utils.py", "b5bcf2c170daefb858b0668b032dd252a379c78332efc91d87c34d7fed2e9373", "826f30b8719f0c715e9743a71dfe48f2ab60f7e0de97e2d542ccbe867b6f68d3"] + ] + }, + "b12x": { + "repository": "https://github.com/local-inference-lab/b12x", + "revision": "b58f34eaf978277621efced6678e6713fd7122e4", + "tree": "7637fe5fb4d88882e0d18cdacc68c493f478499d", + "archive_url": "https://github.com/local-inference-lab/b12x/archive/b58f34eaf978277621efced6678e6713fd7122e4.tar.gz", + "archive_sha256": "8cfd2d8bf09169d00a669f72f179c54bdfc49af35b05214a5e5f0fdb1d5779d8", + "package_files_sha256": "8e5418509749db21c5cec933ac5c476fae4f0fb89355987390dc0a424261ade6", + "installed_text_line_endings": "crlf", + "normalized_text_suffixes": [".c", ".py"] + }, + "cuda": { + "version": "13.3", + "base_url": "https://developer.download.nvidia.com/compute/cuda/redist/", + "components": { + "cuda_nvcc/linux-sbsa/cuda_nvcc-linux-sbsa-13.3.33-archive.tar.xz": "b5dde44aadd52234af3944ae3b2e74e811ad8e71fb600bcc9dfe6d8540353499", + "cuda_cudart/linux-sbsa/cuda_cudart-linux-sbsa-13.3.29-archive.tar.xz": "0cdd73d11885062daf3aa98ad4d7b8bd84f89b398be11f7054edea9ed31f597d", + "cuda_nvrtc/linux-sbsa/cuda_nvrtc-linux-sbsa-13.3.33-archive.tar.xz": "d0502b25799be62a50b743c640e94a1722d20b1ee4ab70d697d71750f04d3b8a", + "cuda_crt/linux-sbsa/cuda_crt-linux-sbsa-13.3.33-archive.tar.xz": "6f6194918c00b980d8fd2111bf0aa004977760855c6e1528e0653bf4c889fbef", + "libnvvm/linux-sbsa/libnvvm-linux-sbsa-13.3.33-archive.tar.xz": "5f8ca5c9a10c3c9804b045960ee6192281efec4c7d83d5f3245ec2de8612118e", + "cccl/linux-sbsa/cccl-linux-sbsa-13.3.3.3.1-archive.tar.xz": "37e9024c5e24a9e9d1618c4fb7b36e74a0a68fac91d589867676952204ecde5b", + "libnvjitlink/linux-sbsa/libnvjitlink-linux-sbsa-13.3.33-archive.tar.xz": "6ed3a14646bd53e25ccf03a52586cdd12b07ad48cf81fe79deac49b5d64c2ce6" + } + }, + "environment": { + "VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH": "1", + "VLLM_B12X_DENSE_ACTIVATION_MODE": "auto", + "VLLM_MTP_NVFP4_LM_HEAD": "1", + "VLLM_LM_HEAD_A16": "1", + "VLLM_MXFP8_LM_HEAD": "0" + } +} diff --git a/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py b/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py new file mode 100644 index 00000000..1d132cd4 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py @@ -0,0 +1,125 @@ +import hashlib +import importlib.util +import json +import tarfile +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent + + +def _module(name: str): + spec = importlib.util.spec_from_file_location(name, HERE / f"{name}.py") + result = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(result) + return result + + +apply_compute = _module("apply_compute") +prepare_compute_source = _module("prepare_compute_source") +verify_compute = _module("verify_compute") + + +def test_source_lock_binds_patch_routes_and_environment() -> None: + lock = json.loads((HERE / "source-lock.json").read_text()) + patch = HERE / lock["vllm"]["patch"] + assert hashlib.sha256(patch.read_bytes()).hexdigest() == lock["vllm"][ + "patch_sha256" + ] + archive = HERE / lock["vllm"]["replacement_archive"] + assert hashlib.sha256(archive.read_bytes()).hexdigest() == lock["vllm"][ + "replacement_archive_sha256" + ] + files = lock["vllm"]["files"] + assert len(files) == 14 + assert len({entry[0] for entry in files}) == len(files) + assert all(base != result for _, base, result in files) + assert lock["b12x"]["revision"] == "b58f34eaf978277621efced6678e6713fd7122e4" + assert lock["environment"] == { + "VLLM_B12X_DENSE_ACTIVATION_MODE": "auto", + "VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH": "1", + "VLLM_LM_HEAD_A16": "1", + "VLLM_MTP_NVFP4_LM_HEAD": "1", + "VLLM_MXFP8_LM_HEAD": "0", + } + + +def test_package_map_includes_runtime_data(tmp_path: Path) -> None: + package = tmp_path / "b12x" + package.mkdir() + (package / "module.py").write_text("value = 1\n") + (package / "profile.json.gz").write_bytes(b"profile") + (package / "README.md").write_text("runtime data\n") + (package / "ignored.pyc").write_bytes(b"cache") + files = prepare_compute_source._package_map(tmp_path, "b12x") + assert set(files) == { + "b12x/README.md", + "b12x/module.py", + "b12x/profile.json.gz", + } + + +def test_verify_rejects_an_empty_b12x_map(tmp_path: Path) -> None: + source_lock = tmp_path / "source-lock.json" + source_lock.write_text( + json.dumps( + { + "vllm": {"files": []}, + "b12x": {"package_files_sha256": "unused"}, + "environment": {}, + } + ) + ) + receipt = tmp_path / "receipt.json" + receipt.write_text( + json.dumps( + { + "source_lock_sha256": hashlib.sha256( + source_lock.read_bytes() + ).hexdigest(), + "vllm_overrides": {}, + "b12x_files": {}, + "environment": {}, + "target_head_quantization": False, + } + ) + ) + with pytest.raises(ValueError, match="no B12X package map"): + verify_compute.verify(tmp_path, receipt, source_lock) + + +def test_vllm_install_fails_before_patch_when_base_hash_drifts( + tmp_path: Path, +) -> None: + prepared = tmp_path / "prepared" + prepared.mkdir() + patch = prepared / "change.patch" + patch.write_text("") + archive = prepared / "files.tar.gz" + with tarfile.open(archive, "w:gz"): + pass + site = tmp_path / "site" + file = site / "vllm/example.py" + file.parent.mkdir(parents=True) + file.write_text("unexpected\n") + lock = { + "vllm": { + "patch": patch.name, + "patch_sha256": hashlib.sha256(b"").hexdigest(), + "replacement_archive": archive.name, + "replacement_archive_sha256": hashlib.sha256( + archive.read_bytes() + ).hexdigest(), + "files": [ + [ + "vllm/example.py", + hashlib.sha256(b"expected\n").hexdigest(), + "x", + ] + ], + } + } + with pytest.raises(ValueError, match="vLLM base hash mismatch"): + apply_compute._install_vllm(prepared, site, lock) diff --git a/runtime/glm53-spark-mtp3-mesh/compute/verify_compute.py b/runtime/glm53-spark-mtp3-mesh/compute/verify_compute.py new file mode 100644 index 00000000..d2a48b31 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/compute/verify_compute.py @@ -0,0 +1,72 @@ +"""Verify the installed compute source against its build receipt.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _map_sha256(files: dict[str, str]) -> str: + payload = json.dumps(files, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(payload).hexdigest() + + +def verify(site_packages: Path, receipt: Path, source_lock: Path) -> dict: + lock = json.loads(source_lock.read_text()) + installed = json.loads(receipt.read_text()) + lock_hash = _sha256(source_lock) + if installed["source_lock_sha256"] != lock_hash: + raise ValueError("installed receipt uses a different compute source lock") + expected_vllm = {path: result for path, _, result in lock["vllm"]["files"]} + if installed["vllm_overrides"] != expected_vllm: + raise ValueError("installed receipt omits or changes vLLM overrides") + for relative, expected in expected_vllm.items(): + actual = _sha256(site_packages / relative) + if actual != expected: + raise ValueError(f"installed vLLM hash mismatch for {relative}: {actual}") + expected_b12x = installed.get("b12x_files") + if not expected_b12x: + raise ValueError("installed receipt has no B12X package map") + actual_b12x = { + path.relative_to(site_packages).as_posix(): _sha256(path) + for path in sorted((site_packages / "b12x").rglob("*")) + if path.is_file() + and "__pycache__" not in path.parts + and path.suffix != ".pyc" + } + if actual_b12x != expected_b12x: + raise ValueError("installed B12X package map mismatch") + expected_b12x_hash = lock["b12x"]["package_files_sha256"] + if installed.get("b12x_package_files_sha256") != expected_b12x_hash: + raise ValueError("installed receipt has a different B12X package-map hash") + if _map_sha256(actual_b12x) != expected_b12x_hash: + raise ValueError("installed B12X package differs from source-lock.json") + if installed["environment"] != lock["environment"]: + raise ValueError("installed compute environment differs from source lock") + if installed.get("target_head_quantization") is not False: + raise ValueError("target LM head must remain unquantized") + return { + "checks_passed": True, + "source_lock_sha256": lock_hash, + "vllm_override_count": len(expected_vllm), + "b12x_file_count": len(expected_b12x), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--site-packages", type=Path, required=True) + parser.add_argument("--receipt", type=Path, required=True) + parser.add_argument("--source-lock", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(verify(args.site_packages, args.receipt, args.source_lock))) + + +if __name__ == "__main__": + main() diff --git a/runtime/glm53-spark-mtp3-mesh/compute/vllm-compute-files.tar.gz b/runtime/glm53-spark-mtp3-mesh/compute/vllm-compute-files.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f0fa7b41468db0e226ba5915630e18a17b47c4b9 GIT binary patch literal 99559 zcmV(%K;pk2iwFP!00002|Kz>tcH>5pC_KOY6u9#JPEl@>TC1#*r+hw$ge11O6#=TM z+Q$ciNJv79BxnHC;+~l^4{@L2KG;0T7fWU$0RmiP_k5fCbV&j-6Pb~bk+H|qWHR0J zXHRi`{`xQfr2qEezYoX9^b`Ke{yaK9I`~mNO7eEjd1=N1@{k$0HVYfE%Q-j&t0%Nh+}C1=C)b8Fy!F$Z1v%WZWX zYjAD0dD@?h5Bq8JZllq5?VdK-(qJ_e#FuSv7`D)4gult+Te}Hm&Pc!?IU?(du5bx|Z95HJVPV z*L9s%$Lb9o*DhMGXg#pZ=C`7jT2r&%a<8m!D?i?}&W0CmyLUmYZ2&XCePG%BUe_+# zNBhV%8?Yi9M|NAyvcaL9Z8Zj#qcvaA*6KFtdt+;p{HBUm9R& zn-`XAJEmg`=&CdMkR8|~M2ln# z+W~y;s1``O0WPm`Jx<2qA}Q!0HfHVqr|sLl6!ae413V}IS-dI+9i9x{y_5*uO+pV3 z+rfTaJ8ifFK;DMk1|H%pO-QL<+q5qMOmN6-v(s-|gAxGu`)=F(2F?2Bz-jS_tI)hZ zbg!-%+3mjBuyPcBKijm|>ATJ)(h%D{Yjv9ifKhF`!;afKN1A}>+r(!puMh0gXu*lT zE`uqyFde50Gi|pVn0E;b{$_QZ-k@>mez95?mk#1lU&r!nYaA8!=-B;oY@U>~`Pw!G zMbPgyd7}Rw2D3_zYE+G#mx<>RbjfD3u0Hy)pY_(0^q~ayRo(y370o2x4*wl{s z6)~#?Xw^BBsPg{cP@~;Vv+uO7Ef*+8>%7(Ao1s7Gowwjx0ql3IuMQkPg=i-S9}ly* zHZZ@rJT=(0rU_@@((HDDdLq+sD%pr%c+X*-v(adRKjVk#&aVZ?P&j4}=GQj+_^`9; z+Q*Jx24=qx{rb7)**%i2uMb^-RbvP!0PIKuScYaP4Qv4cX`Kxn7#zDZUR?}&!+u#8 z+&^yGDINApZ*Ya1HEcUAfP4cO0_U>V)IfI^Mvnxtx%WI{pwT03A8p%$T_%Lu5Z}Bw z_LEpgHhmCr12)>0S&D6~4$}K`%Ym~aSOkS+X{+m2)9MwsI?}m_VY5}z>_BVQfh*a} zu2ZM<+H1mo%<&xWmbQ{Jy-ZDpfQr=`S#oyF{N7C!6DG&k$F`c@l6Hq~$vj=afIGbQ+B=_a2c z$Gj+dd4*D{*+w=2)+Z#xrbVpH(##tj^U4w*%W(ir|8{YYJ3jCyz8CxA*Dpcx0IyoJ zyRZ-o0I+90yHwg|y4mglIV9Rk_ZD$1#qBR0r>`X1bZP2}FpTew*atal9B1haY_0Wq zXmuMYkvsTsbaI?+h4|36BpcP~J89$n!(3b2>Ge@40aC77<(eQ{dy&;0Va ztQD%AY!mnd0Do^_yMQl4E2G!hUm~JB+4I)Gc7(jS8r27I*#r7AsE7R~+|k{O)nw|h zFR5=SXKS7z5d+%PrE4@rj!(ufzh=R3`NcIax;@)zHOdCGx;P%np)Sf&NK1jo9t=R# z65K+`&RsOSlAYkWX!PUu;P3@$2HltNGl_yE6#Q`R4iyV>``=J;w`xy&t?rfE>z67% zI$zKGC$0q-Eb~H5paF0(f@GP+t?V*Ly+=rm`nFOXWTq9|`9Kh0mj@dS>LRt@uC067 z>YtShP`N&xp0#Rl)d+#zC$^#AYquKT3NU?nWH=Mf5M(d&1gR?6n4K=ymWwLeu83)Ama>_Gq9M!i<(MEpm?XFKiv@b#E|8hK*Z2)whgB|9iPxD)#f z)V2VZH;iG}^3F9{EbW8|0$Dn1c>l zgXEc}Hu&F8{20u4s_EZ%3#9Pr-;qD@Uc>_;`1~Me*M)zYOh3@-1iw4I{?&^8W$}L{ z_j=XpwA`yB;>uyC`dRk8Y&T>tbNH2wtOGz^XL>)8Ihb& z>(uJK0zoy?I06VwxuwdSw5d6Cdd{$GZKwq+>PcHciiWj{H@$ni4Vy6pvaSpkOG0WA zCFl-D#ZpB>c7g=A}>e{G# zvrv9%0Y(1XN*4ktJPuITjoy7t#pbT;*KK`m1Av>Bvj3xVVAdM(5N2#m%UyIn`+dgP zh$_tOMqlcY8T?1xs0<9zc4Ox|#6X9OBK^Bo-&q0>IpJ{yZ@#9HBW83M$emO1F z-1SOzyQ(=^nc>p9PK{wpLYHa`8;rt-dBJ=uZ#HgiDpT&)KeGKz_CaC68sM#9TNfy7 zlp((AA<%~wfDViZQdS+T^14$@7|FYQ0}*FSx=79DqHEE6y|&AKFMOKR2?8FsTL=rQ z%bYTrUf`yu5nz(-@=LPwnTFJFR4zfcY%)VqGwQTbf%tjYBIhZ( zLrB6?bZnVNJMZIUfh9N*KR<|{$KvM^!4Gbkos>^Y|19}x!$eIKN{1jkpZ7}zqvf8; zgC&b`t6Yk+{Q;bRq`wZYbg-+{!zdD9o&lj#_ejpC=lEoQiL+1&8~377?kL&oBQsH= zvf&$#jwui;COyQSerY+E?&Y_$0o-Tu{H)eEN$+HvU6zp|uTkbUdLSuS*t}E&VA?t- z))=0(R_o4@x9yyF(290p(Mnd>2F0$CdhJ^WJy^;_x@_aMd^7T|P3a4J4%~({SmSC= z@Bor9aE7F!DxLO4-*J03hz{T2vJk|)(?GA(GqOX*zA{6?ulZO2x*=M4A(`S17_4)u zkRJpgr_<|xlOF~q$r%jEr?!-!h^DkGw@ISv3Lwx@{%WJg#9+{4L7)RisX@+T-%7n! z*$7Ah4mN6Zgg5328*lg8`MOmHH~`i)0L>;KqCY%|Q|!o8CZ_k!{)%ET9t3jpzU)Z( zA-&h(Fs^R9lUJ1JGKhYqW+V^M^s;qvi5yOEcyWmeJHEtW`U8`cpQ}|eF1d-ou5}yV z*!17+UboluK?^036hS&V<(BKT2Xjl(XLZvrupNj9PjlE<$)cb2$ScUTJE^;!&MTu` zZqP-)u*M~aUThGY(HwUA>kR_pXt%mJ$KF7A^s$J$nx3jIjb3liY;}=cXj*myRoBqF zocF@L;_#b1QOlnLY}D#D+QX)`ekB(T9W*)3KS4wZ%IjpMtU+^)8bo$#+SkYAenNU0 zeFR}AEhR%mUt_*`jXX#wCOfc_uDDnB_+{*fu5Dghc=Q0fba6hIdC@DfM~m6q8$J4C zA>8xx>}~?iivXJ;h)-*jYiRl8)XVE7bPL(F#>E3 z;8?GmLYMa)UYjdOuIL~fw)kk;jX|pqhr%w<4SccNhf3+Pc2;r@2rl@4ubhDznCW7h z?R7j2`k^8Mz|D0Amx{CE;>x#Ne+Ee>#+!?mhkheqDu+})2a}0` zYE@$zj{S)-^QOM>{aiYgvPCJ zc%zX&Py8`<$LP&>)Z(3S7xT%q}^RajELHu%3s2-(LA*R|K0yDXqhNDk&7l2K= zz>eAP)9tlCyQu?VIa`XpFO2of}O9;3-P(jUdPhR5Sf^f!VturU4Ua<1S)aI-Jv}H@NySE?S_$y0oEn6DYSZTJ1=ON?oz#V7bU@+H&V3Wp9-yYP&%yd*T?>ofYkE7 zz6CBze&uVR1EaWjD%{aS7>xXw7l2QZlb~~2)mN&YTz(-sX3dK->BBF-{8~+4{3?2Q zu+-R}#X%B0`I)P>`W>NjrhEFqUr!}N#vvb(58RC5853`MJNAGX@)F62`LQ7YH1%ir zGzxG)V2!+B7AMS#0kyFL=u#iS& zp~es4 z0+>pe1~xWSV}``jp5=2E%#;YBE6$sPB%9F zmOMx)X_VWrs}rA+8$mlEVk^E=@4knH#o^@1m(Zr;nY;sjJOyh7*EaEfe>ITC1Mt(Y z0@c71EIWUa!@yc#AUQ%sJQb%mgI;3phzhw)$}9kgyU7pckQ0QcM8M8IAil=f* z+Nvn!e-mjRz%AXHs3D8$0T5yd`y;|(gy$E5$z>EL8>F*?^rwcyuf)M|Ijr67< z01~`BMO+v!Dc*!W6YesRRw(k{L!<5aBuGk;XT}>dH=Z9rvQ@KRm&h}Z_5Y@O<$L2=Y@p!-dz~ZO1P5=TAdy?z2A5Ld#ZY)H z{dreY@%2zW5SSxO=c+Is%2VT3ofI|L1{OXbh z+QOSFPOmPY6mT5==@%G(M+0)K-gr10m~MR|%1hQ};*R}1-4(lsEsFUIxj#zetIA1K zbKu>r?C!2Z3A`m!1se*%vdyhUPqjC|Ik6e1!n7m)v=d9Amba-oovP z@FPu^e-D4bVNHjH%Z2pvq@oP*u6>46DF?AYB3MGt9-`X#Cj#3~Yf z_u#M=s$P^X1dl8Y1DJa6&-{ofHZpg9f@g7@(Vi?~Y3Z6r;Zrb%Ef@1wd~q3R&4EQ~ z9p28$;sGyZXEph-NH>@RzQ#7C^x_x!`CqjTSza#7hq@&~X?Ep+E#DJe#eBO-?FRz3 zKO6sG0Lg@Xj1bCvln~B<_zT=S7Fom=g%OMa$J2jFB5%Z`o;tyl4LSk%ky`EKUu!kv z7y8gCysF`=8q5Q3RU8hOZA>%kqcKY4n?=yK=~>04#k;OlcDoVt)@}qR!W~Tl7uEa4 z>Lb{G$%#|?TR6BipeXDKV31iLbK~%>v6zAM6qf}0B8C=3_97YHNu+CgaP zas;81W|H>ICuq6v^4e8op}5_I=pmPMs-L<|hukecnT%?6LEV?{qOvgsXF&#|aFOud zA{p8#8Ddk^DZJEd_!4n2q~e$&9vt950F?q%UeZ&QvKRnE*c>Hw0cTQCCh7x@;92-P zntg2kc<+t_J&7Oug(p&(K)u_TPM}NHJUFPD5*vsLOQ5+NfMQhmo><7vZf={YZ4i@z zPz}Ro5qOKP5pkTpsN%z~ZtrThkPNGK-{o#y%%I&3lk;#f8yD~#FC!|-~iQ~Pl8d9 zO!V!~SSDeofoK*ghxDl8%*~F*R=jEc@g3_BJ}v!cv_ROwE_4+*!PGY_p!e6vt1Y({ z!#G|92Nq}>@nNV_1IQwGJxBfqkI^rfH^-=G1h6XlM&W1*(#m|ndVrnf{B=90g6%k6 zVxF_@WXhghc>?4&Iv~PPB5MQK!x$GP=2&Mxox6}7b^5Ok+U{uYh1~`j$|x9*eaUka zr_R&nSkbTf%xsr?!Hm<{A`WP&_tfkgnCl%7MNN;wXO0emEl#G332;gND4V3#{DU2TdNVp^km66hMP6Lu1$Xr=R=0S|>^0=-VS6g^lDnM9&v1M_Cu3B)aN+YNq ztxPRgEz!h&A~%|97K|2M*T$`vj2>#ij1CL1s|YhVFR)M16j&B?qrwXvP&9_0_g*w6 zej&UQ!%%-z7R$sH5gs5Rssoy}pi@fEl+OWBND;ptbz7LNp)u!2dMY~V3aK31FYVYeu#-s1gLB3TK-Kfm4N-5^G6600ikSjHa4Shln%l zXl{LY=c6(Qriw|xQJsTM2L|Ogh;?wo^xhwdl)8d^uv0pEh8L2W93HP8K!?S0`;OT; zGylN%milKwKScfk4gs9%u^*3M*Qh=OYF|I^UF}_yGMXhXU~&L8fu{~5f;xgz!kf~a z0i%qiDaN&Q>mE_!p41I-lw#xZi*eXAZYLoK)gU3l(1KWe57$WIPmHhq;ZL+tvv5{( zplRgI;Pe2=9Ygcl2+m&3p9FC5NaY6CCx2WFB3vVk1h1huWQ~u)1?1RpWTMdzFdobd zx(l2Vgj?{3-@|gq951gQV}hb5PVI)%yScE=*$o`;(W8$yIq(3r*vA4F#@vs@APNmD z0)Z#5+)~H_uzp-K{FuD%{$$+T9NgR>_)#qg%MWKiiGglz_Ooy9?qK$s_olgb251ey zntGA){Dg*@bg-NtRYRwLzyUpFWhx%xQ}!(UcVetd&wq9>zMRD2&mx-ikL6{{i|JjA zr0-vAKQ{mb@heOoIFsOaW+ld$l#_2LFG6NX3`NQqx~8D}jsF6qA@N;uZoMH)y@|c) z99YjJgpG-B$U+j=#a$+p-8>YEwVdtMsLUhOSb9p&12Z5y2~LY|T22RRRG3#pQyspN zL}y6oB0QNGQAl}IH*6nqhAvIyHt6QWt}d;RW`2_)IGMb(8(fsR4y^608cT_6vs=>f zUV`ajDwU|f;PB~T=>I*uC0$#zAb(}laZe_2mPO|?#h1ZK;dBPP;&d$7QDI1wxpIrS zGWC{ct!CF%BP9<^DTKI#xITLtjmjr-P|qRVjr_OF_B z+`&f@B(Fl6s%!uclF;4^6z;tRWR{jip8wt$jkv~N^cBU6ySv~8aZXAVY;o6?vL<dX9|+WV%<&j- z8-wiaP58K?|F9Pr!4$YuB}$7 zSwgf7gG}3MnxH$wY-V$GZ#JZAU6ska3x*sH+MA7r4QXfeF;v>`bz`IrwJ|^h?s$(1 zuCH@HJ)*+BM_EAWVi6g@>6*_a{uVvWCof=64P|WKByBh^DW7_L^_{-dI=#?0NmRa$ z)-t056zDdDlQ;?Qfwx{ne4tZJj@S#%aK28&6s@vS<@d%QB;Rg0=xmEY9}`qEWb9)G z>hvT-*3|aoCRU<` zz&0IiQ_yO`Q0pN})-$+XK&s&4jciO)mQ#R))I;&z%8UmH=n`ncCsGKXa2m#d#m{Jr zVj&xZs|PPOrZBevWF}+gCn#HyJvqAbM(OxEg@WCZyonV*~S$LT-hQ+?Kgna)SE5fyuB)=IePCbX2FOLJ*u5G z0&-gHMM4(naEhH#DvcjmRidDWqr_EBGm;J=1rM64$I{YW5CMyeop@+`7NK{EY%Fcx zQuMtA{%bahd|Z?iQqoVb3!?^2@)SAEI_F4`k&_V9R6Lh&CE3|PA=K=n>&C4EX;m_@)4y_-cGwEC2B6RSAB7Z*d z1dC0g(^=nAK&Sg~jswQV<+*X^#ds*-Rq#7J5<{u2y21BCc~X(SF8A;jmJWNnH;NW< z=#M7hV!WrC?!vJTrjlBi^Z5h?S(LA5uj#i0j^K2pYH@i&VvNeo!`%%??_<;p`~IW* zbaa^g`xyTGumAktyLD3eYSR`Y^k0()Jlp^UWV5ByjJSKHZGLCDHtQxj^=yYc?}O^h zNO;tWqAdsvV7D|K(-cksPD~v=Vh(N>NG|cnVJc4jfRtB;)h3K{U5$II#({Nw*v}&w z5eq!n3y^#O9*;o+oX^xxFYYQ_!6@mmGiB<-4w79JY1U*h_T^|u>L$Uh>YNIGWa^?g z+j6cyZIjco^u;6*j+k%*zY(F%fJlPTO2`So_fcfK^~p8Nz>$umd^B#OaB=^T zq?A?~&xFS(x*Y-YojQMM2ZGHwdpO!2xst_u2Zu+;+Sch+de@0GnYOt_5~kcNHM@}x z4mn;wiC#(g4#e$(GZZ7~(2*$xP?%#=Up)6mq|HZ2+~n#9^YPlrJzcS^e9PRT?k3(n z-y3AN6uMIoSINXPAEgFV3?WSnsSQ0(@0N4jZ+C$e^e4v=6+)W5s>G__#i;zL z-A8b{&av1daERgT9f$#RS)eicUI_vjn_jLL+6hrV?LWbx2xlxa)EDX$JrX|Sy#O0Ce}FCYnGi(?_fX(xT|hdAegzc~w!^=o0r(;= zoIYt!@G^s)oS`$s=s~DGf-x=vxQD60Ev2|e@ zK3sppwcWlZ{)lC2Nlows@t*jo=<9|T3ojbSUm|pgkMfhqRH576Ua&j_(}Vm4N9uec zrO4t>p1s$Y#i?|2{H$yRD}1bj3>8e$9U_j{8#Bb&iH6Mz53oXm%$AZlX4_uIsrGy0 zix8CgY8-RhqWz%#XAoz&{%k$Ib!eMwogpWa3a4$ zd!2ZbZ=NHM$gdzSP^>Si(I~r=OfAMS(DxEP@~WrWs93Npm(6`EK6u42g@U6aW~vFn zIO%6ENF~l@0Im!9m$c+aws4J$AwA(U^j;Y>%7>kQ1sRUu+l@}`#}cK~gSYA&~qaBazV+@z1py_5~J zSe899(CrNejkT8cOk02rT#zU|!%nAWk2Xg~6`C8_?u&Ri#abbv-8#-TqpDr*UBJ}tlucuW zHL9p}dQA(SRS_`>hdlHoTqpjMKPi|?+q$;e8_a^+Q9*$79?;wc$_JIV&j&af-M;LMZC5H^q4$D1AY$a15!=W)RVx%&na z6b-Y!BQksfs4Gbo5+3XVT{n8l1B9dWDrqo1_K1bZLQJYRQR8C5w~7#zQ2ES_cZu*B zm6?y%wf4LaDcillM(Bv`R3M$YWJ$~b!D=xJURVGn_JHIfT%g&tTl{&la0+$Y$3#N1U#E!4pN)M{@>Jz4JdjUPjyOU+pi0+6dW6;VH!1I-xqlb&czAVT zX|^t9d0m!*a<1H&#yLqy<2R+R=J~{nA6jVnho|$}qp?Rn=LWl9N2vWnX{IhjV$Fyx z35~LQp}7Byt_30j4Bcpq|N77WrS35NC!8mE1f_Oc#k*C7)k!$V<9Nb$Qxkr*6%nC; zfVz!;{&cdt4<9PjDXY zF)RiI?L;`1dWnDksw*D*TqhaJQ8=v}Ml2##w|Dg%2z#BrbWV5ATHWS~oiA*@bd{`FaxCMudCBQ|jntNd zo`-3B#hKpvh5caB9xXP%EX?uUJ4bmFWtApA+X_F5`me;2q7Hu;SDs}OIMfy7{i_=- z-IFN~fJO@oAaCNWpta7sDy*22(obvgc=0@dBYbt8p9@tp;;6H$>+RU{rQU@)=gQyU zw$SFfYxqF)qRA_!3nE57+A)SX(0HU)pdG<#-*&n0Gwq%E3vKzMA;a{;8?kBcuU(CZ4dAW z%Y+$Uj?EJhkHCGYN;g5 zZ?1~zXqp$Q;^TV%@(st3G#~xWWljiMVU?%9>}X!J>j9fwMKJ7#v*7L!`11qt_>+3^_MxD>KTMyWXUEPNhS~k-` zWhbK|?Q=D@rOB(M2o~3eqKIa1AS-*k2_8k`tcFPyHZxqz8(|oY@rq*I4cqHD;sB`e zp0Iz#i(3w2$8aMRJ8eBQosY^{O614StefQ=@^V7Sz!K_SfWVPrgRlX?-&?tG+&(FPr|T@9wVnnO>WW%>c8-P4$XKaZZO8T!3=Gc8E1nQ z)<-=XI)GHR=LvJpx3}EQJ1~u26VivC(Iu|MAVzUt)ArQw(d45f-cB0EhM$zIDDQV zk3>%En0!sQVA;0i*rEo-pAb!v8Sf9oQ{cr#b6pWxQ9`VreWCN7_})xfnAky5SE7;= z%>P`P^d5C^S#9b^d;gRFMmM1nu!#~=jkaZWw?f_0K??EK(G13SzI1z{@I=BpLo*Ydg(r?3aL0kC z=tFE&7@w-VMOCp#TC-y5$yhraw6%ysA!O}>{572?uQIK=%A+7Ym&v7MHCtv8*~#Wk z(pqKI`!er=&`+UST&jswh6(zsYxyJas>16ub4Csh z_l?SXa~x?<%s}Aqn7>!He8<7c>%)O=C1#!bM$1pTqyP1Be(B4pEh3+n zmpO+F0EM`Ou>ya0^B4;dLQfkU=hgUa;fgZHQ7F)X2q^VFkoWNBzex*!1 zq#OAiGulLWtdTEFJt(Dj$_T{fFu^2C&ooTx8Dl*7-duUPk;XKOQ13rv^QnCmV8PP# zRK0TB(^Nh}u}{Bd0-@C2#--!*(cO;7qqvh4ZV}=zlnH%u^c)6J+I)SJp{=O8JAl6j zR>QHk;tR@0DU>h(TeL$WwJCSPZqNExV~%;&fgp@0sjSK?Lub`kdK1uh&k3P%7aoRl3IqJ$;KM%-o;No2``fyv1x(x;MJ z1he@9aG|2z=c5M?lY-2BfAoO#n8XAi9wC_kFqOn9p-3h?iX0^IbEkWI#BRoLL!MIz z`2-WBX(U?9Y4dk~qy>v|jj zY#`%R>~qD=PFCNg@FV;$Tylrq~y`X3VTEmnIJ~`=hJE2WGv(=l3 z?(}cpKkki3=)=kJ(fi}&i)oqLVM4C#XUgHnb{w*mq@%0U%RF5bJN`NAb6 z<3z0R;!qht3N#LRddHc6`gM}Qv(3m`ZK%&}hIK3rIj2|%OuJRuTKYr|p z^;V7l_;9k1m+c@gD#6aMj-cQAYPGjIKAC>V_WXGA@%@L7KQ2+eGog6Iqj8N~ztQRm zuagO}bes(Vs7Z}z63On9`!E8GR*~*CV+Zp+jh@h+GYcr$qins@#vQ-TfQcSKTS?1m z9yz{<7x}JIsn4-#Qd1frfmHeTW0`swB(nffK(PRrZ|dg%WMu`iL*$!%fm+XRb=kPG zGktYA_RS^H$0dDVw6jau6eUrhY9O3(Q)I)@5`%b@u9HgYQdo(cCX*gaZef^!%n{C( z>yJpD1rP&dkouBVM$C3u&0d-eqOf08ZBFhm68j45#f&r?Ie=>0hD2;9trnN|;H+rU zTPY_}BHU3rDBQ&DHcv>sy>1t{_}-fRqp!PalK*#YVQ$Xrl{BiQa5hhsHJ4ueke<0T z+lpYVDN#v+DQq6H$L)6FJ^DxK&Q&?Z#3*{5Cn1@7;H+Ucqk;nId8=<52gm!hBO<*M zO!T~~TWdWO2u};!VJ?v|M!O1ZP#ee{-ya;7cD!SX4GhhP;~FrmM*Ip7rkZ=7!JnpV z?=%*HfJT$!aEY{g6u`+Sv*)92a2^F@tN=OV6`$%#!%oM&H0?_`CuaMiH)uJRoeeAv zMZL64+ydfyB5wIe0kEndvRx)_-A3`XZ6fy(Bl1%q=IFs+4r=XT$}h$DU>_N;6)#2) z=xFBjhZ*e@v=PdsWuNc2*)wt@{xOTj%fxdtwe5?I8l0wx4l1PD-%bWIe)?RnuxM73Jnw@XwvR?nw%j9cg;!tTk?}*Yh9B4kuagr+PEnBzGt%>e_RRsTrPXa(U(xIb>`uFP(Q@o9B=wbJ zurfK%DP3JF1Lwq_-D9~4#2|0%&GFFIfc6A;Dwmd&1Ss7BiU+VxOehtn-Z>*L!lW^R z>*O932k~6;4*{w z?W=MLlMZenF}`rmzzT)31=ml6Mrd79J2#xQ#WeLLYQPT6W%AS)&Pwe9$lx*nP(-iz zdp+IXlY@_kocAbW)M=W1r*&;%uGZFhi?is0h|)u)ip*x}wyo~Qrk;%lMEw}dsVucE zt{k0M&6cjZB~BntNHc)O2Fk~g=F6bGER)w%d1zwxYnYeCp`R4yEo--i^H>?h1p4Tp zxxy4IW*gT6D-dCsO)E>W*Au4|!))T)!$2hkYrEIIU|lj9VF8A^=MO20brlC*ABY_6 z?1pve08I?eDuAwre7NqZdK&Baw9eR6v;c?#q{pmgrgw;G?_}!9`rqS}jdhe0C#nmn zFy`tNd^*tSZCR_$gG5V5C@vSF?Ob{0GZD04VJ#kt$|nsz{Ms>{O~i$gp75>lVmf`z z$qSzs{z3@rWI+PHE%xqwrsNFG9URkNc^%e1l3Y5Sv8tw!&PoXRcfezCGFT$R z%J75EjrV?<$|Zh)Mvo#v6p(ztq#C* zW=N@N4ew$ZywADM@(JnC*$JpDdOgiwWp|m=l15;42+s;l4%1YfF4H}7jq95fgg0EN zjt8i*@ru8XSrjlQ%uMqap5vwov$fnwJ&}ZIJGOalIab&14Q%)G&>C9n5=~`j(uxs8 zwa-mL>mNmaEHX|BNsz=$PuhVCqtUUs3T#iQ<5ofPWn-{kN3k|c7Rlw15ovh3ncXNw z2yZ8*d;n2RX*f9)PYfo>Cpm7*TVuCwv{Cm(dl@b;T?wNR`S@zwm(qmI0&h|a?`pC< zC<@@{4G_niMT&R69Oso+UDvi?2fGbxyG#T5zNo*#@z*PT;c&s) zVg*tP9}woNy{Cgck{5kO1_2{Ishr@$USIwR+pc) z?ezK>>)PsG*lUuF6!(D{9`Ed~nAt5{7SCZcW@b^tc$Fen$NeQ_9AXKmCaSiZZ$QUP zZv^IHsSPy=(RaIS-Z5HKrf4mz^`0dyBwHw0&xRRaLBhv|o*GEp9D(EZ4Wr z#zQSdOv;YGvO98Rr&`43ag?@9w~aR>o$r-_y@dgq>jwI@Ant2jpY7tmJYm~UQ=SuA zgqU)h?!Uzh6*l)XVn^EHz+@>~^CzZs_GsdZsnWLCO>)r^86Q)nuf~55hF!dzf!!UL z?y%nkA+UR~lv7IvIzyX*W%t^c&xM!XYz;PHtjoF<<*75JJJRJj<}VH_@vcm_$|ab5 zRD4d+jUfXG5%DWIWiIxuyDFm#1=_E?d(O7aXcnMi9{}|R+qL3j5$gcaJ$t_^&4jUMCti4m@6+At>hV32&WQkX?}Gs zkj`?kQF-wk-X&+Z7iLhw+6N&5{zsva+eSB2V`zd!%PlvGL0NjE&0bJ z1d&P}R2)tg&sb6^6rQSWwFtKT zl~vE5GDRf;(whf=GRKS%ct)g_hburG_E{VMXhz2 z5KPUyhV|?=a_+dDgtvR%$J>MVqYwKZ?tJgVkKX=|hsOsWd~e_P4i4TQemL+Be|-O8 z|NRMf(E^P`Jd>-3sKtB2(h2i&8Hny82_|uU{`!6UVE>>7|D1?~7;ufCOU3uVl>Fu` z)!32%Sz(>bHOwT&RXLahr}x_ew@{QWlySCfBv!Y9OhB2*YZfP|ViYNbPB~Cy1A-F+ zRjwNiRX{RR&jzF&kiNX=N+&-|1k;GqikQ(UFzTZRSfk#j3J)U?wX@HxmkGed(;x0; zW3VvsZW9FkK{xB5k)GRMeqCm#mO5_DGcJu##yldbvLnaCbIrS-g=9>ac(2mQ71)j> z0?wLTnIdUy%3XYyUtIH|+q0ckW22h5`91|n@QD5p>0z8&=6FSF=JU4cVgxvrrL!8t z!2p;~;TX1+zI%h=3jKra^fJ9@c7^X!-dn=hxji_1*-0b0=M#*M07pQ$zaATQ=iqRE z5C4^V!ElbkY8X|8?;e-mPJ7>o6F}`h5kaMYzW+R}3<=%{$8%k_MDg!M1}#h?E-VxN3+Choxu}P#W^qxX zEvUjdQqCr@%g;VAu7i%H#(bKPXHVT8(o{+^mco5 z`!WZF@S4VA3C8{1%g5tGW6$Wcj`sI=_4-BJ#h)&O_W%Vxj(u_2;rS8v32IpT`Rx4Q z16)tY^d@O;4eiJRGbolyO{qfeJVYBr1vL|XR3kgB_{6R z^@qY4xK?M#E)YN(J`YWxyyjBY=@pKRYsLU%bcmNNW_DLzHdNh$5@us!-^~iYXS-~D zPSCZ9x?)}jI_;+9ie+^yCIU6ZEvG(doUfnLB`Dvzv6soM0LY5ST`ygx+ZftT55$PK zELiKj1o$;8N5Pw^!-QBXIK{{ur36FMLuj=x(xMYmw;1^o(jh`0<#48|4j2_Y8N$#nY{z;y0@w6O*SOfHJ;1fN!gWZae+{;$~ER&vM%c;l# zwnpS!VmyZFG{jsoaY3={Vs%??N}ZndmX%<)ZLx6xH-v9o9T|4u42e`88=cT1r_RMa zTVK7C=VPOCH$Op%FnFSyVAr?@~ zPc!m;&#vpDrZ`DEl&lh+Cs>zv%3|w07(H?$`7K}qFj^!Psc@REEKEmm5?UhR-c65c zL{L!7URrhMy?&wo`G{qt0XN&b&Ku*g0t zTB^4P_zg~asttWg%3Z;WOuWTx^gy-=(#>MZz-nfAq2fh{J3oqq>G#^L#h%{QHE z=(g?jDfFjCH9IQ*dhlUaoZk*PFY6Mq((cqnxDfTBSfp%FYYF9_6#=#-AiT9}H z1|jB2)%d`_;ivE{h>!RI1Ceh4y~f65fxg1&EX8!fD05xRJ*tYH`e@f;OgFI5qK7Me z8&+p`r0zSPNO@Oq@%%+RCE5yEt2PcWSVfEnl7t8_6kWLRH$^i5%Z~!i`~AH}vuhsj z68Tpu0!?W_)0PhBHcgKq@{_T1kE8DB8D{xABoEQ2*=JP$?oTqIrx#B%T?K$L-$2up z5X5MnN7xe6hvFIJk7Jv7+NQuju$KD^gknxeN@z3@F4+I`dO(=Rx|QeD=d`>%3(( zF%vDjc)VpH%Y!T<-bR|4GBUBC1hgV)C?(8<8?Kb@Z9+4KHf0C6GzTzm>ucW{IQEzMCzo^u)j|aU&MBd;pD2#o1=q)bl$;>K_JlbK%SQRy1@;RoN{?4uW(8Od+Y!A}#?{Rc}OpNf4

TIyEe$u#jbXVC)=@S+?>_?R=EC+rS|L#^DKhG{!4T6J8f2j}_V z!{#`?W}LU#Hn!V{dcOFN2)N2bDWrbQn6`<@U`0lZj6bZEd#pBCrIgLvcaTW!df7MyJ4~P zHKXhG)c(Z%3TB(_m#D)%78G5=tN8ncdk3d!F(Y3%c_XM4C~2~j1P|C6z_+^pOu#RV zUJUpmAPVCVM@Cl@!7DR{2_T(FmZq!9YjsZ!Yb^|aEW3<%mf zB-aDN45{hz1&oA?&-Fo?3)-^nZ7b!KkFu&_o_mIBWm;r`PyU*`o+bU^d$mFXp!7Wv z>G@S6SRw1kdfS{Wp}JIC3yYJf?1XLf1?ytEuyF4EhLI51~4}azrx%oW6YoK zVMeE{5w8E5JcLWJfZxk3jg?$tI!awX#E97e28+KX@qMOuS&(&R9az$Z9QbKLqlrqx zc0bHylJ$p zK*Bdp16*lAV!grxJzL5=1HMnnoh~v87LX0!&h*UZHgPdFa`NQ65rEhQ5W7fLD!b+7 z`_KbFKwAMt{t_pNjJ&MDMs4^EC_iI#02+@2_V|LAV{-V3g_kakBc1nE6+1ZrGGpBA zVK8&zQofBBKG-+9Lob&dZd%tATfWJb=ey+hypRpl(+P_xN7#1Cj7pKUvf6^o8H#BR z0>B$De&p@cs5-f4P%IyZvdTi@GLqy#Wl3Pu`W=>0Gg`!#Q!v4Z2pA13bF&&A1zjud zFEx!)Y@^KID<_}&bF-5q@tH{&(jda;JtjTMOpJ_0s!(j0kx-~x{%fKbr-42xuxd;x zEyoQK8Wkf3aL}25PyT3=S9B!xMluT3an2n!&S2mU7CvO75|-T5^Mf6@ApZ7;SRJX? za2uDy?$uUt*5v~fhc+(9&dA#XaRaS$TI!H_5=@1doXvgRHs!^&33}?y%VKBJ`Jfww zv6#!?12l|{xjds`s;R+{Vt}qf%4+VU(nu|bSw+=)NEImr~o!GxI?M@mOyryZ!UQCP6 zP&xCP+lK=!%3(mWEi#K2bw*sMFJ}x~h_gPXTvDU~%L9x8HhDEfL(pqs6vV#D?7);# zj)5Y38V>M4HElqa!0%M)8G9Y_4#^{XP9@S$Iliox`p`(-|I%V6HHI?f{qKZ4r`uD!}Gd2t~Bj%xw(5MsFv zjwE6fP+pxfI|_3FbVss%xln<@<;aFjbKtmkkFK~(WwXqiRh1934RiG_2WUADRfLbI zRnHzxC|st+#hC2r9BUr{*u7d)5<|AiRbJipyE)|oT~Ye@d}WB>;`;I|fgf*V*!Pvh zG~20)2v7%p0H{Fxi%aaY>;78To95O0VFZFQ37OJAY&|p!$bG@Y6iwHl+3F)N5-Gbl zUhd@nGo!$UjrX%-+Nk|+^;h6irWxHWKi2o*e+TKg>0?OQpD~esZ{Vzlh~9V^%`hp} z@4if7E>fLPHXL@_)*{(RLnX7`WQa<)+3TPj2VZLrJDqPpFW{=RHxKJ8A3_wE=J>0e z>x?LOOKCmGL3{~%Tr5>k>`hAlVQROXQ{}ydJ)3=fm#Qi|uHnCU7zjw8RAmM!rA&>( zy`d*fpqd#E)ADEo#@dK*$4xOhRCdG%c&Xe4$xm8-$s`I!i8jmaXGB6PCUZkB0(EBS z@uGcU?#;CEZ57N$cNT}DBVMQrUW_Z1LUGCs&=Tv*Q$P0NN+trLX4QgOdY0UU3}XlIJ>Z4Rqa-B zC`{>HP6VOV7k*F39c0!Z1@{RE2()lf{0%SL1;#*q;W|TDou1N)x#PL3TDzh(Ze%Jq zpgSx}uCdrQqk<Q7{`#7I4|`*8dx}qir^<4o0qZF~F+Tx3}4o<<7{+mnHvF%oZr-s?sY|gQX&f* zcAQqurqo^V=7!WCbzr*j|5%Ri=EkVdB7xmYM)h6zg7rd4^iPeQ&bM~!%&qnu%-ou1 z0Ijcpo>&yvy*39T(}jgePTWSXzcH?_8D&cVBV!CA%z|_UoNrI%PfT+po{91 zo=_IfDG(ZTo+Y0nxu9=RK90O|O9HId$Q+~JCYduZq4PZXZO?oD4M2^-*#&7LI_99= z>as7211wX+HJi=N8Li^(E0`<{LrNn+jIq*{XJ)uOL1N$B=7xod3pz0bv63b5wPcUK zo1f(CSUMakf(b=4&I;)?AApt3CF&4~>wwF|NzK;uD2Np+aDbBqkdzh5`7`TuZ?h3YU^Yc620wrHvL8Rf^~{^$O;Xc!^E`W}`9e zlukWsd{{Z%?0`kF54nkQnrDuJ%fNgZ`f2WsgDU%WFLguj;+IKW8OaT!ndbj{q=(~hIUb6W2X2hzv_Xgw&gIuhByPfgF>t18*Smx-7O3oaY z1jP6u9P25p(vW^=n+ABZif=a7>jxL48yRDS07lw)!5&YDi^&`v>m_5Fmet3px->l% z0kd71;F2*4jaV9}l0`I=%FA$(AZd!V61&$2IB46S!AJ7<50ZBxE1Y-d zr%0-Dhh+F|VteKxX@4o^eClE9gK!bEvMr}lr>OU#gT!gMqVJjmR@{BXMIvFux&$<@ zp%9@bK zWpv{B2zZZ7EtXf@DXRufr1@1<~cakQnP*8)nVmQE>Ynh3xp2NG_A!CdzqLL#FJZuH&Hm{YEe$>KLMS>?Or%~lGf|Jri0F}vZc zHuk69Oq3yg_M+)xPDVA#XG^K{AAL1|7!QUq*h=W9`XLnt%xTs{mg=<5MW}~Wr31Ya zel}EuE=+3VTVq@T!GPmc6;&2_TEPslMFx#!Qs2gj7bR$h!;}}4z-ts@kyt#?qT1hD zy(6jIV54>n!8eZas0L$pG}=FCwbh5YuL=PpRjI7fglO z{K-%4dOo{!p{}oN1!^q*eia1zM4CG(N)C$x0{7wj8FG>yr7z~7Gwg35f-H5>9EBMo znx}BK@C)r-w$cmlSMFJUm|s_ixG}{StfQNR^T*U?h&KWUf@p!EIpTB<|1_D=8}x>K z_}%ICueK&jCBqb_jzZ+cltxr-X|LI*xJOP(M5%zqdBf?GjqjE&IGG(jh{U2Kk4)7= zjM{;Wd{m4jvPm)!W*mh?hG4 z8vMTd>q8fA5PJUAn*P#&AXzFhzQ2r5P1D8IX2k(PheDKp{XivGbjFU(aJI z2|u4qwjiT^o%l%7_LH`qeL|R&JwJpKl@lkm#G-qZfmzhU=b21CjVXrehD~|*5!f=D zv(uMuW~ah8qK`jksW-H^YGkr6B;O*RHL9n>>3qR;N3j|LOc3~8fB-VERXv9g=}2mD zMJMErS7C$tZ3svdFQ&0J9Rw8SO$sAHuq9;W@^B4+al1gRB5$k`8Ua)1e3svr0wG&7>^Mh6XXZct%@h(8BKPl8Anvs zMmQvMp$t)B1M5@3wWnRM$h!*_2MDUM#E1Y*2X|V!vb2t^yau5O=atQ}nG($A@GAxw zb}<1k8kj7z^QP@88mDlthQcG$?i}pzbLK9*9&6@6cg|-YY*>jTX;Ju`H&dmqQX*3* zIZC1MgU6K(l2df|y!El56BQ2u7NvXL#=LOjaOwrKB9PE+oykR1N|Ezj@&vhTu>*VM zVwY&Pq*fXb{uzrN*jq1pU@vj!8R?U7y*0z6`j|THb`p*rYb+6w1VsMPoV<#YX=bc` zlu^MU$!Xx!&`C{2ClM(X*RtLDxc?WUav9F&m^w{c-dVfXxFT7H!UAc^=WWxzY<17A z!B&{QW~f5O<(yP?n3T~_*^rnEBV0(8vKcn>?BMV#UG}IV;GWjQ?OK}5Ei?H{CVKwP*7&OTD?CX9p5jLAW1zDt9-X=QsY z3v|u}wvXXt{h`}CKhGs*EoH3oT{dN`IxnqRa;&H2TW@VhITvCRC>VWf6;8nzg%^y7 zfZ;GsGEQ-YJXxLqpTA(8gQ*u8BIYh)4X%)Dq^<8xl(E$^;rlak=f)L_s)q(J+xx~} z?OvDS2(*$&twcJYu)L*QdSEM4ud+K)t*>&3wAbWB;5W~pNCQsv;K z_Od$ea@{M``D&&3WCSv;(7*Rz47~_5Hl$eoMH)5BVL&p;ZERE|aI4CAxT{rS8~Dx8 zMsdo+6qpp6D@Cj^hMgf`uoNPyNT|p>g^j<=fqarh5F}cY<=h(k)8MHHV=cm2&%CTh zydS|S>RhhtziOs4ZiV6(s=$i&)lebEIkCM4$ z>finDN9pJz&X! zP+ssCmEwtX@0f=zu1fng?6V&{$)T#bIcmj2wN#tR1X4ra6r_Q3jiC>uZ~oiy*l_ z`qFVymO_JbJWDdm!1k`)0#UYcX?D9-+r8{EHINobAeVu6aR_&5f??NWJjz6COJCO;1qpQZorK|> z3Jf4ay8DyV%E>rLE~QNF_}X1T7+Aw0Xd-ba0W|m1;!$XGA&N|M$}pXW{kQFIamRI= zy9K;fO>&!!E$)o_`s2aUz80MroU_$PslmJ0_*>^Q-)hJdrYQ?6%z^lrnQVxO0dXbI>MSkRR(d#<4 zanY;pM&8^KAn&LUGVp(~ziQ#z>}z3PKCa8zR}i_Hptl=R&??^_z~t;`T`_UYg=SH}!8%Gp{dIBIB*t2m88vtt?(ud$Ceqj2!5s%=gz zt7steM35^B;5e0chm{7%k}u-dx^d}G=CQ1C?Ir1iDXZ4~jNI#hr&jb_U!}-qPeyf2 zV!+w(G1%P$;f<7;n6nM-G<@EI2rcD>NPzkQth;qhjHubPw?^IN5sEhaTm21BlE|yiVsw%-*Oyi43h1($jx?!^$#h5)jE5`%I=$3bdo5UOs4qT zbT)gMEGG)pbX2PFiKNPu5~1Kwu3jgmFtPh4PIP#V54OpSvv!;+sx2!=ACHY6H~}F4 zSfx5%&YdLO|M0evZi^}qg!Rgjak-ZE_r}nHsoBPrC>P=xOFw#r=-VC^v9f$xe{JiT)k7ZYM4&cD- z-e46@H15i`$`x9+sb_P0=4KH1u~hEM>yofQt4@`fiEJb%@mi53&AK2ooXWI(+UPRp zj1Yrl(PrTqE^=?>d^}NfM6Secsd9V8;r}ZmL06BHM(;{CKDC;oxj<^!sU%zMON$F7 zeF8=lRr?WZBDsitrha&@2f~;+z^rhjHdm^0tI_Lj&UdaIr-=KsW*qBwx0sDY%}9|r zQ5tr1^M^O70W&I`$Z1^|>XN$=tuA}R0PtE3ho=P$PJ2bU@LW>M^8VXqMbFVTkkuE! z>v2N#3I(|Lzz@cZ<)459f>6=arOO<)Q;7Z{U)G72o!EMKJjricr!yEZWf{RZtSZG0}f zsTAL+1N5x0_YEev6V-XqMeb+gAgitUYQdFYiE!&TV)yDnV7W)jO2@MGav%i-EwcD5 zGd0Bziv-mK+T=x+G5d1pG<#pVZ=L?SVTz1mSJy^E%T+HfG|sJse^d4Yc@?0Q0gZcb zghhls2*GK7oc3AwvZ4KT*=?k;3m`3bKf#snl7%s@8kh7J)u19UH9pZBG}EI)XvS_V zZgxBZ(x~LiU=cBK=Gdf4R{$X8}mr2krS6IHsJedw9-Kv8mWi#lFp3G}!V z{+znR+LveD28eo8R*Izu4?|q1c#*_XrXo%QpZ=sx0~N{PoG=H@YwEbn-mqt41&R}L zAD=kSr7NaT$sB;?Y69z+*`@LUr5-i3v4}pjr}PUho6_B1lIiS{Gq|3$#rw0i=y7)0 zh9`M8*S0&9N?y2C>$pd{K)`lSR0F6hgtD7J(HTc3{tPqrPIry}`p^GG&_=8Y?iU!$ zs!E2bo=qEiLaVBwx^qt_^_8S8!_eC4L7I_?65mk8GFC?6yc4!;O~l+0X$UN2oYIbx zzB$kj5 zz42IvcPRZ^CP!bd(??p|HG!*bby3mTa<&VbEEz(#$`Kij3VRRC7KSj3ZZt@-L2rO{ z<0;tsW#m)r9^X!xU8;A`t|e5Oi&E8TbJnO$GIHOgY3ssuV|dou&iGt7 zz$&+`*tqwiTVyEQ8Y(reW-(~%o7X!MbN02N#^sJ1oSw^i-Fm{bT1ljNnoGpUl!BCbxlZG!MVap zl>N8W&t$mo^-=G$I^Cn*Dw7;leu?0rW$`3+F{cw_Wg3+!SC~GgUAU&2}ea+~q9)E-H&f`mPo0ENWd9qvqTmNjUOl2BL*RT8Q^P_Dhx1pH9Bds#|=X6lDnw8S9Ia=N zXOhn#hdR{jbHZzkow!G%HC1IjCjk&`738H;MG7Nb=z!{^pUpi1!R!@eo11vDyP@%RVE7% zICoEt0rVBlT|BoC^L{U{*Pw+`9{KUILlTLRR#Vs{DBSEz%eizfznu+QO~seK!9!`& zkqcuJEJ4$x2q0zl9SRoR5pnqvZF+NmX0|Y(feh#|7EVaL?Kpr{&^kQD?_$}5iz|rK z`93iOX^ACD@Iq{s@@6Jwfh=)`xqwurd6b=5p{SL(6S)brN`FRWU#-!bjSJVLGBC;| zbbF4q_r)A^TipvO;T?cDC8vYj+@sIZi@+#4HRJ*+3;m9Tqv+$V$lgO5jI{do1RZ3g z(S(VefDQ`DSl;DXXocm*8xcy6SzW7q z$CF?pyd`Nmi62danV0iO6c@|DvirTRZP_lM0Ndu@az7Ofv!W=5z%BIO714=}kfiJe z3cF8-RLBEuI#k;M&&`UF5&cU>LF566~#Ff3sHo>UR0N`bKWuCzIkC0z1WUbE*WE8 z8YrCa{FRHr#IMt-k6B>#Nb;i6cd5aN$~b=O0IzFxwk489Lu``5N4Ej5o!lEkGfM|q zL9S`vDXI~33CLWDQl;amg0KBX%dZL7DnN;Rh9QA)(8@}u@fMWLjZhj^mTk9%39_Uk zdVEvz-#q$)22l#B4U*K?fMn-k|gLS*#kXXgc4cb}K11Q*0GETp_DeI4z)yn@k9srDs}&etMHrLDmDE)$KI; z8Jm#InT{$Zka{H7o{N$qgJ=s~bxdKs8lYOuvA#N6M@y|5t4ykl44ITotMFOy?Y{lq+jLIbn05a}^2z~faWhnBka9^qbI@4nkBQbB40A2H5 zck6gF&q9vfnzFwfDOZGmg!;f52frw1n~b$`h`G%;fv9`lYd5XI3REg;ovYL#QlUw{ zl#3NHgC{a{%_rVWBt{qY+NzutCu3nlkxCSRqRYAlviV&w;Sx$z7>VS^7-Vsu_}^Mh z%3U>ay_3+LGo(E3-M8G>nf`?*_}ZHU zW6k0v6}fOW5Q$xiVVOs0(5^oFw{Glvz}aDKvZug*R`yKZ%8S0^_G}xtFWa$%K^>@? z+cwXvw*9Gm=#=M81rvAT-J;LJo$(qjB4aw5Gj$Z%@}++y)eR6aShDaOS@Qlb_P(^a zjU!2PKQluAp$Cpx12PB_5M|5sz%dj_$$a89q@?jY2BXykniPh(ga&w6tM}Vq~ZBiAh&U)v`>iYMq?M_X?h2m?;xd$z_sH7*F50ZQ~ zw|q$$;z$SL9W%G3QzX!7Yuz*~m8kU0pyHWvK@`Qf(8Va-buN>^Fw-k?^YtToE+9}B zEK9Q)gsHOQhzu7+C}KMQxk^_h2FUe3xxRh|Hmb}tQlS+CzO52wrrF%~8G@O{yC)|{ zOdeq6oDl(k{%R?3UF0?sHag`1bYBpTCCiD1H;CI8P~t8Ivc& z`y_i$Y3P*VorBPmEc$k1^#WAVTyRdSGb@c`$t{QX8YRiV=?Xyt!p<#Z5`-H{93r=f z^Q*VweEm_|Ew|1759nHfr8XV>E$0pF0TJ5M!z(o1+q|_|zyypJ#u^lC62c zRJ<-)w(SsDCO|^$wB&5dY-gemR#^$wy2kxSGoU&PC4&?M!%=h#FGpS}n%2S(Td~ag# zGx^sjf-63HyL+;~b^O37TZ5F%cMi#9x$vu0N~@4YNuoHHUIC~N-t52LJ4SQS4v7b! z1#!=CeqSHTXUg#!0^)1iZ!&3-#*%%o9+MKuBu2M+zF!#*vGe@s2j)7nJRyo4i$XWZ zH_Wm!ZAP7SI92GdQWx_OyzRFe@Z{uZTgV+^RwnIv4v9Vv1h;lj!Hw2T$n6ebW$gZT zqh^OI3C1Xy&u8^G9)OqNJdW!%wDZ&&Es)ZnSaMW*KbTsPPH+ho^l6D6Qm8j#yHt3V zkc719`AsFGXZ*(+4j-V$Z4u5T)dPj+c>b2OTPmqEsh0P2`N;hMV z^z}zUU8XTfCRuIs>8G!QW9QBi-}j^|GLol1D8YB}NZuo}i&PyP0}-i^SPEoRf$j9vqyLCW|h>9-Eo3%cu7EX)u=FxurMr3*wMkpQWZl`v#-k5+_{ybu8Kkyxwp0{ARL znYl%g>+$V}B)+}AkwZ!gE0++IHv^TU$f7lH>CAcnS1pG;V2pM;8?9(-j$s%nR+<1t z0}5#?eh!uM7*8&aAY_%NxxR;(%>FP%;R<|70WGz{xtEY(dT~dK(5#sf8j!U)db+yt z^v_V~;s;I&IGC$X;a6nFl#C0R+zfiQX_PW)F|4!uaSu>&0@m&+_?N555npafE!5lC zGR#E|yEIEN$84HLUmc%B8||l^ZxxA41;`a-y9Nk*R3w@_2%GU?58uAwF*QGusoeq{9)7(q`k zszKunP9blhIu}sg3T(JsCnk?cO?n*QnbURhfHNkNhkMl)W5|Y?_VpR$$w^eU)TmAx z8c6-ZsX5sfzM)O>@aVM>XiC`@SEE^i5#VFW63oGYz21U(E43uY2_JsgrU>>%>3S~& z0gRf$Y*6S*iAQpe#i_3|bf}T3T`@kyj(XbLml;n^x8Cl8PU~Q+O0ZbCZ&Ph1x`)YM za|oaNnfsm=*?iMsgA>T`kk(g7J!mZ?_e=IRkcG7EJjHK2vJmo{!$ZTHqojDN!|U54 zdF#tUsxF#sju#QuX~F#b$^d8@+yoB^WQ(*i;?xMPLy(O7A*e?M2MMZDpa-zslHB7* zBw+@GF{+(X%1(+G$}a{P0cs@NKYGc_+B<|w1xqV-yaNPuQou}0ei_cr>MDU?b#%E{ zB?&Hb80CW(Z=i0%$=>$qk{Tzww|`bF32`n}a*l;!Pq}|NOm$l@44k9Tvf}=Ickks- zuVvD0$gv=qv*W!(kPDBNG+9;}u)aQA^mZ_V47f5?B@n3xuS*R4g%$)O zc~^3f+(n?im<^_waTP2`VydfJrby&8!Fj1Sy+`AAOWeU-@TZ!?8L~2e*HKM}F*K;O zeb)ZI(-eFvSm)mlrUy98a~N_9FMGBQ%ACTGt4-+=!NL2xR(oSj{O2h#(Zu#*Qeln| z7(s^eA4p%g826B?V#k6XkyntOd6s#Rp@iC>=TO@hwIU~b$FGl0DhK43j^isieKd~# ze!Aabc2U_thsyj@|FIiwwA$ad+MRZz2`6Nn8z5CEsg^-UTr|ojK&vs;U!j29DexDN9AkbsJeAm56RY!>VCyb1G`6sI;{?G&(>%oEUxLV`i4 z?v1NKe^VZ#KN|qxSplBqmD_MsaHYA%Nb-M9R3o{0f#6YOkhWg$J>T2kd;O2<#jL|a zaKR7GP-c{qCfqa%fY|<9Ace3ij|Wo2!G&NsRLT=ii(FxBaFEV4OLUqx7a8d9%e;pL zJ?SK)gTW0rN~(GW9Pb8kNHSzC5FnRG$88!$*@JrEszhZ307_qDo391eoDTbWDGB2P zzTjA_2fhYT=eL5p2W7kf?8GdQfU-Mgrsxm%d4q&Vem1lwItyJ(;E3!vUmj(a+!h)+xAo?H*ID>i7+Kf4(`~Uao>y!7ydPR=3Gu zE`zG_*h$b5%s}12h0$bL1g%yh*DD93CPvAPHGl+`xO1|#cPMiXoIZ4g)+iN%05y6` zQT%)}gR|o((IfqBk8UH}n4Svsc1K!B&qFS+ObWdOBxTSlN;fdG22ZZ1=ip6~0ZB5X z*z2uMyS1+COz^&DpESs4CjfH*hA9=ozZ__*jOX|Bf@Jr*$9m-kRbM8vliZ7q119hA z4W!UGe0921j(evwB%A7l&Mxow&w{<8O`3p)VJ@haxJ&iwHpg$|?FGN$QkR09cK|Ow z$_z803%Hvu@xW)$?UpDi6~@Q+W91&fx17wHkJb3@6fZBD<~wGWSBeK$53xfp|(tFgn9USFy|lC1w?)Tlr=@*fv$qLm$9>V z0=}GL^mwbY^%-hFYamaAt<~ypZgs!U_@)}I{KLMwf}io_e7G>n!}mjffqwJKu$(Dr zQqL(RH8axt6^o`nPqK6$N#H(Ez`u+0fc=1cynvPmlU?;tl}?e?AIiX&r9?henlEKO z!d`HQncUg(C|@}{Z?^UyV(Si#QO-!g?Zq_@Nxv*)l9N+2X6y~Nz~J74t>a_UDaj3e z@-#ObKkOA@Aln2~_BX(YWip2n`keu^#gMD8E+`ABakm)iDz=HLJ&I|gOa;VB{=WGa zs!8=@ROjMUpep2Q?=lJkphg!Cih<&uE&-Qsg=Tdm8=M8;@-${(eOg&rfxyjYe_36P z&cv)YMXyx=khmBx2V=bE&qX8gR}_nTdk#*Daa@O0Yyx#?P>z=xXCQtFMLxV-ubAC0 z14w1I!dA82&1ee^D@1f44j`kb1x%cKBOSo%6gnnkR`4!@tQ^=2u6k+p#fhr2Y7I;h z7MO)p5#Wd9$>PaFJVQdY7q!l@>b$9l2Sr%^D(Fnxoz|j-ibGYUh@VFG7U;G>u?0LR)569ueM?vVzEKN zU~C#-@a;@<55Tm_qF>g+0VNo&hLa0H|Fxbg+M07mRrMNTD{)4>p`iO}y1soCGX-ss zi8L~Q5;S5}s(17JI+|Sj%+rK4SsAK5h5SyH$tg8LN)E!abTtL*JBXitB;mUa()_S# zLdRN_O$;K_zs?X>uWh`vaFV{Oa-VQTZOsdT%BTYmOL+v*WZ|D3mp&Df!LdM_H0>jm zqZ!C3Uz|mjPsD@nA&kl~{i|pTBgznQ(C%jXGUT?cb)k!*Z&Wgwyt`-)Ic@7uLI z>X_JAk@TN@6GCKQMHZcal0xOjU8j7aheU8@RV!dPR4?Z=zy3WX7Y3|PE9=_I)NO-o zGPF=N;RB+|jy&KySdzjT5avR0;#qRj7G^C>u=BDuO zo2LTwgYOdA+A5>+MK2Imvm?gB7wVAnAqLA1agP@Ea_v!bR-1Zx81bxy7Oo7AG_xDE z24@Gnk+}p)AfQ4Zl=?oB&;l z`ysIA3Dmw#X5eyPU+OD=h8pdnjfA!gX8`Z0$t)w<&>!^XX9CHZ?B;x~knKeA6vUzC z6$?FHSAfBMDV<{EK9W?ru*@05U_V6t83-hTj~!5C#XC;iASd^Zq6;8Amqx)-LI@r0sBkJ}-pKF_&>=8CvI!>y7Jn{|^hd4g)Yi}O|2cZDhski7v|}NE zY@JG9dFw3(s)3yY!;#(2f@S8|Nlr%y@qN1|Z@2cHw&ZKF^XA~#|0$~OXB)g!ZWZm7 zVD|ZLmx*46iIG1>?8PJ{pVlt~xaPr>}zNBLL)oT>#U zv}bG5*Gl=q4Mp2b8>^R$eG^kYexwF`j|q%XN*yt~GD}He&&GNh?WZFWl>lje-gz0# zFm+4@X@};hHkI@!ldP7Gf zsAjKyQn>c;E{rD*!NEVoNjZM=+C7l$K)1_kCGUdZ6TZ1pi0#(V>zH(i`Rmgs@zyrx z0e-bBK5f4{-Q7OfeXTXI-tQjn93Mga{hO10<78}$ouvx(KXA@YLhQ_g*_8emgmUs` z*M`ZHl0rxt@;HL{Nut*Jr>dQ;Wi>qBvnR?V3j@(#Z-GBKZoIo^M|vxF$B#sd*YWF< zt?gIN%l)I5SORf>_wDX}@U6_V9C$5tUV+z)=H9K(+w32m_}=o;OXx5(#8jqqG-Q5L zr?oeK*K$<>E3{VC=9&&+7K`1D_rzMLAd@Pa;+~1uuaDL1SG&8%@z(y{TX#hzb!4p8 zrDa>}x*h7nkfooezq9Mx$-M(W!56#WccwDnTW#J7!g5s7X6wz)URj$1K~A=0e&z5x zl~NJ2AQZHF{Tk@po0HQ(E1C_A1}S9rc0=t1IpJVuN$^4~rCTIbQWY5a=FqcD3&Rn z9B$;g-nR3m0%E9VwqenZ1y(~62^^0~T~X>!n?pE$6Tf=PUiK6R=xG6wV6aC%0{?$^ z4+XRY_OeTEsNHL*TtOEuI+03ue*YqU4r*1MyKHWz!1+kyi7?AL_$%m7@f&aVPWRCK zvio*#+r;C;BjaNEL^1|y@UhqwXTe#_Au|=3%{d+F;FmO}^oi6(iX)CN26D+KCHqpl zCS<7ry;?|+Z4Kt>DAV;FLYSq#7!Yp>L59qU)rZD%Vi*lOyEmIa(xWcVL37rkbX?JH zo|Ox95xq^S=ayWJl1~+NA+Nn(J=#_#03@&lNxJ$`E zGV%)2AR9ZQGJKuqVj1ScMhVRLGiUTK?4q^rbpb@y4bGDLcs$8C@eoSdU9Cnfq^zPs z*4m@@^Z7K}TwA*mLiXYUMDaB-%qmCE!S8Dq!^y>3@9DFRi*}lP|J}v<`gh+y>9+;g zzY`BGpS7Pp`L6%ndU|=0Y;@M9Xez{x@pK0M66qQ^Ee;{xSgd4CK)F9i;;ZSRHNF38 zzuo>_XOoQjpEs3@@4xGJ+Wq9B^W;gdbJ2g6_P^_2e7Es^@~o4z)2H8WbUK}jr@bYo zvi|$-k(@^7cb&%jKgKl1=>i;saV8tT>rCbhkSEW6*J(iNr{Nz3S7#T)u|1F4$<{y2 ztkj1`f{TjF?%_Z;!Tg*&pxaM^VOYM`79<+kiBDcXYooi%>9aMc0wHL{ zL%y*te!kt6#h(~T$c89i@*xBhhfzoPZu0ssrJREf|3FWvvIqSND@pI7$> zJs}9Bt9z&#y&R;o&1h?iiRL@4^%ZUfwE?y#)B72CIloTt=9`iFlcKx9#5f^GDpiAt zFR+yr6w2cGashGnU}7d|46O^~8@#f@&(ONXJy47@QM5&l)Ga+cnnb)7Vry)OdnCCW z{0yaaC>tvX;!W)-1%d3m{>QQLSjeew)Z<2YHpbT=4^7fbV^x)0>nFolRbx(z9m^M5 zTUrhrJ98b+#go4$v(aMuW<210IFK0Y#W?r2#sb+*CcujfW(7V#F*&NooJiJO zTu0He49$@jsX z7ByhICuDGmJAKjot_5Y-lWAICZ8ybV`1{#v`@GT0D-W|+Pq`gpYyv{*W7&LDv~Qxv zs`0Uw=n!9~nDsns@WqaSlEySG9HCE5$wOc1I|i3xa($UDn80uW4&`%5*n-s`)PRu0 zC8jm1VBwr);GxgJM>ot#p0rwNUO#*1n+r-{1)Y{Ii$oV~6*&iI>}LKLkhBZ`X=>V@DS8>iqwXlVtG5Xo*vy?7k(Z;=AGco#yW4t z1(_isMg6gkcHq0v8;TVo@o~;<-FcEb%NziuJetmVM>r0Kv#OESxaoccjA|zsz z(g)LjDbInw;v^^y#tPSYKt~;kZ#h_dLk&*sF&_7lX+qJi7(~T~Tx?5Tp`6b9VmeIE z!1o18d`n_*zNyuh7CK(J- zyT(~dYgIlAE+pao)00jdO93N(272i&!^(Kk!N0>o7~YT zFr~e*;Mj)4yi%JC!otJ+ZVIVQxd~inXYk$gcMhEBBpS$54vH4UD7Ezak59T9#QBbN zZ^Q3F)Xp=YYW`2T)Qmn}NaPfOOQLMDfS{R+#bC(bpre$h304mJwU92ArvlmYIYv8t z^m@rSR>kq^aO9%iX5a!};$W4{1wPFQl$c6pF*>=%oEoAomU-C>@Bx-P8~j+;INzg? zQ^0msl&|EK=<Z8-|Q!}~}*z!^Tz^B@ZyFcbc#{Izv=Fw@IU*Z*wIG&)h zV-649KJC;(u9yf0tb;>bBA|M$VUh!;zu(l|qgTey~S;Ek@IE zY;Da!GCymk*g1I0&&>7Hv$IX}5Nv8+J8$*Vxe&AKji#q3fy3#~^Y-;^XjLzYBoZ+8)%qxuj`qnp+t4h^S)^cK|)G3RWU z324Gan_tRmx40q1Ams4IOW7PYAf!|NXTj9P6!^W#j00X4tyW8`>c(OU5-d3y{-Lq+ zrp9D5c$!Ou_?$<*nWw@Jf!Ql{2n1g3S$6pc2UXl}E) zeu#PK){w4SobPpd-*sUHYeO#51BuYGia}bX{WR(K#b3*IGZa229EG2TXBCn4u8Ci4to*Cnf5tbL(~V!z z{?mE7@zk;Zi01#X|NO`Pql^+U6>N+li3qnLhCdd_8KWKOcel@2mYEXf^;fdE6pgCJ zgBoyyaW?2>&T#oFX}rPI8$80{&5P-VX|K^9729i&j{Me!8Xa;~b@(3rUaowSdU2=( z(#6ZzP#P(ndllP_HWuaaxhpddEn&?0g0>rJz0d$-*f8YO4da!0{c9g1JP62za(!Kw zKhYLRLChq%bZKasz5YTh=*AquV?qg=QSL$}6h~KDU^gkYXgSs_(w7{*g{mXV8?PFS zLj+oZI1GpEcJRwDbO3CcneXA9VS}m$Q_!AYs^MeNNvyt$>RQdZ6hI%s1APKKZ|#XW ze|vg1?zq}E`&}}ezE2F{kvP}xogp;JZ39%%DS+ol@GswB+Xooa9D-VRo_LIDgdt&L z8rnr{W7>o5Xw~dze-!f>uAKn%yPMBEbE#@RTk7JH_-bBUxH;mA4BYRKQ0KR^x9Nv} zNvoU0Y62wMg)5ugbnZyz&if+9H=4{J%=m_^*JgkvTjp%4I&;6J8BXI$yJ4ht?L(@U zPs#6ULtVbaD#U53Nmnf83xBM|w-TL3YFrQ4d%VoU(g(bRJFUiPS=VsTbm3S+o59!G zLYvmX>J7x*TEl5rfb?*1JevOIi`xrmD*0gJrqlvo+vZ=qtba$tLS0SlanxxP*%rn{ z28DlL^}W|>U)>TLz=0B@G&IpS=8vS)ziu>rDtf7omxnp8i&kAJ@5=68&9UDcz-oK% zyIk>VPG))KK3>S5t=u<&Jt+uE4f^uPDd3pstPVx3n zyW6k!4qrB-t?|8ON5DSmV5}JrW&)3A$@oeRh@~TCdVsO^5XhC19b}7RWjk2WGYzQH zNG+__h-n}}V?k9FxNn+V2@we!#8gECI6^Rqtk_u#WS6(%5v$enCVeP=jw${6vK{!e zi5mhLxj7QTn2xqAARUV(Uu$CshU6CV>%rcM?bPiA)~m2_MHyxkaE6PkD_A?qo~1E7 zNM>4sx)_Frmyo)2Cmqg{!*sqSiyejsqd!6L%-K9o5D@2|m;O%Fo(MYBr0wUfQ1Vql z*pGu2kmejv{@7z$Dl$nylqG)_lor_oEmHIM1F>OZR`dJTr8qtBHF{0(Yl#!ky6z`( zFwDJva&USm211DIqiJ<336zYLVg$GkSo_E$y%LY;)^iYbj+MjA-Ix!X5k3Fhh$py~ z`e{#bM(TxFW(*ZH2E(ECChN_huwQ(!fGSLxxyLu{Rt_#zTNAzlx1K`JokN9`2kAUP zug-7yMO@(0b*(H>emElrwR!*mB&+I@c7F&6xdJ9+7i+Gfb zf&56*UaMv9BEfrmGF*(t$4oP5S2-*WO1SsK?@v?>cH`;f)^6O86Ts8&4cEh^&c#?e zz^97)p|=<&Dl^tkfv`0EkYyU6gX(;O zTu2nhNya$I%?Qnirs>s4HO^T}Wfh+n@OfAff1WaiBK?P&taM0;fe_M;kuA5P9YEI_ zg=Sd%h+3FK>sw*lb595JJ-`{P5XWlnz5#q14z0)J@)FX<7x&R@J;vvaPFj}f#hA64bPc8bRlI!k zV=2Bp!LkSGY;|%w#!8mRKjrkAZanhWQaH2?xoyUi4`QERY^=9Qb41gC%TsIEy1dma z&mRKbB-erGH-YC@f#-mFP3>#S$tHUII9gqAJvFq{`5$U$^GArSfzL3)k|O}P^Lz+J zdu!^l9FsL?3muIE9W|uXqF)ty2YT7)=zYUzXAA)Z)@`CwDJo8e(!n^iG?r=dIQ4!y=vC$C$@nY;RBd zhO=VtI~r~czhlsruN4Yi^ntydpM5Rz zpfCU2j!t%C^)0(e?x}MMXq)rB3Rc7|3M3UG6y|3{*g<9R1ZyD*tQHJDp8RBXuPCcH zzww}d2iBlrYdV`;&61ILP-nAYZ!*lBckM>>SX)l?4ZKfK=T#SMY5u06ePjMUg)os9NFDDCXlzP#r5cASLZH*ykBmefy$Hpw}vjt?#{jva7 z2un4}g}Lc9Is$gze8qq2S{T-d{yVC>Q~WVnH-;1VU>K0=xS@X`Y!?$#M*_xkrr`fX zSL>xM;C9fTzvu12udJ=*4M{K<0L{JE+x*#TqJ0h=fpUm@Xr{Hl;!kI?E4jg#KWdC` zY$oaC>_&%o;cb@~ig=eEf7in-(o|#mcD+zbAG@A@>QbRSs z8cLcx|A9r1n!k~%AhHBt7GT5-G|d6d8#Mwfsu%b|ls z=-IC$$^#+t!qiQL=-qbbS<^Qfqo+oUNMIr*jFg>~^0rkg_VXm@)5*|0$@~i2BB*<} zR$&r3iRtdLHcEx}q^9w(40{CR>B09Hhp>K!(Fk`rNNvE?OsJMZO06Xa3G|UG6FinY zEsnY_U$QP{%H~V}6A;!-D_=b~0LIDPAY(#@G)Nkz&x#Z&P2G!DocDb(EUpbT0});g z7`c@WQ13Yz_M?p*P)JHVYVWXLkX9%QX3G;qQ=_FyqIc3@X}`H7e8J2>jRcC}%99#G`D#!n z&nIx{v>!MnqdmAi8vRiR<(qEjT=Tdq9%GycB5_;KH@}&0dQ;nJhuC>d?}XSEDw`RD zsi9O&kCN*&z67(iw8wc)vVfyF!%*rGd`{WjRG=fwOJiafLA|HF)oK}sT6zb-(BSI@ zTO~v%9u)ZvcqA@$XmOq&V9)d@_k??({X0<7h#)@bYwPm4)(1~pXLAVzT8uNn45h!M z^>%>S$=T>hry^rhqOic7P@I!@$#o$lza_2B?aIc47X#r!{MtWcYUy6Y# zXbhvkwVATm%X*Ekk0aO&Sx6H2F|857asY8mf;r2?)kPQEyLX5HkL8IFTy{o2U?ZHv ziUJ@y%s^Mcnah5nHB<^5MuN0Jp=GZ82lHq}`xH$=+&O+fFY$tC1HRGlsd01#l?%1b zRGD%DvcWvn1V8DatN3$uTw}D<$8pG$m7Qx%LSn@QyR=6Jq7V>$XVy=lC}ZdKtw66V zc(!4JEzjtfpIov;dIq(2d9H9kOpcjg)MJbK)fv(Klsm_liqmV#&wIwSH~Tp)6Z zjZ(5{ZUz%kLy@g3fsbB%`$|HU?7!Z@XEvl@+}QC=eMR5iC#>;*5ZwWLNFE0 zpGkgBd!%_hxp?&z@^jP4WC(4@sR(0snN_3W)U%2BKCoOe90e{k z9T8gO_%(<6;Iu5aay#~oz84_=N5p1Ai(iC{IB|lcb!#krN zN8@qx-uuP)I;Ll)<`US_l2AMqhG)%11?AxyiJW^MBCq0cI?rQ=Y7OQ(Q8nca&#L<$ zvpy;vV?dlTEK<~tz`@+hTD==Dh8kr^?|k`&49--uI{KJW#~+qSx%S7op)Ey@o~XWz z#nZShc*#DA#>E3LMY|krrn`s%p+j3&z8=&`juCzLHGz&&riMYv52EdPGdJ zC)_)SH6^Gkfwjfi+pPZiW_X!aT#vR)>&vDroFEE82r@?!B53^K`Ty@15VSi%-@s8dm=1V)R(676MvyHoP=6;RBb#rV@`$% zo5sDgj`Pev;9wm`-CDpl7AFB9DV%ogQl^>iU_5u}Jc7#OghGB1t`|gz-ZiT0ZY6l_ zSVhh2$-fJliU%fphrTfa{IyzGDQjy#X;fpQn>?I$D+5578gzNSEiLF?z7ygX$;4Df z{0y$}RRIFgQnRw~c57p5ItKw*&%infTJateF|T~UGiBVbov-*$oYh`EuQ~Iy`u*8B z)Mk1cd}H0`;zJbwT(^$AF^&dne4Lv`lm*LJ?PF_NzA3n4=o7Kt+S4({Q}o4xTJ|1t zL)kRRhB?%8xP*%nltg%u8^ZvVsMp*^;*8-9+*olgK)11Z!+jOKO4Dg{87Bg1F2UuQ zk`RK47y~TGUehG&%rxmosUT6&>*G~r?y<(3PuMRDM@ohQ!P7pazsw7QkrE^6qckg~ z&LNUnWGQ35iuT6+1r(eWXGzFNz4IrHuBxj>l>OL|8Pv3t8Cexp^KuW&_`-EQZWVHZ%<%wFqKgg3TpBx0$t0em4V+4GjgaN z@uXzLJhn{YsJs5eDn^6%ZTGVFk$(GR1E=P5?yZD*K2YoV^UhYiyR~_aGJRaPFQ&`* z0$Ymhs=7g5jPc$NJjnMs(LNNcf_G;}d+>?`wc>?}Mm}Qg>Uj|CAWIDWWxqL;tUkR0R zD}ABJqaF6{6G=fIgk}h^EcWbR8H&-W!U#t1^+PDxJNJ#>wMmH0s2y`V<<*oJRT^O& zbI+qWl=1$f>ve47WHl!^`zL%!1+w-9DY4J$SytueQ`3JO#R%j8`2Z@QvG^tDzxeo* zO5St$eXhdh(Z~M47stWK2M-T!ckr3ly(5Q>#w{HNpUI`{ya+yjgrxaSbuqp0cK>uWz3)-4Y z-g4gO0prrQ-U67@gErZt+W>6_Mtn4E&Q?>&IKv6-9`xpB7t{;WSRv}?)2(#N!+)&R zR#uqlF!bmf-C^wA@-?v(yA8vdWvw@lnV`G`s0Yh7tV4+_oE^zRbnB`j8J0x@$V0aY--(#I-o5CU;laaX6cwmGmF`@fO zt$_n39#_x*;3QL!BB?(y1R#Rh)TLQPmY9H`WdN>OCbs$!%qvFM!MdnT$m0_bP|SOb zUIA;hIeGgFg(l;Q2du!N{fC@W-9(XQL!Rh`IOETPwo;CHtEBLv8ODhe&Fq9{0X=t6 zcCMh}3GvO#ifDcwTCSy5ivCy22&7e2SA`P8Z?XD|;??7*lXyIt0YP-qCDzR~YSDz< zS~j>EO$Pm%eN8DFXEGgZLb6g%85K`!c0%(sVXq4TKHIkHw*qMcYo2Bmr=t4?RA`pF z!&RKP%s3UE`9z&PJFQ@SNySI;(foDPA`@3BvSOO;Db=@^H+386M2tQG+_~GX1r&AA zv$MLE*sCvVI}4_7jyPi`o-0wskemIRQ9uah@ro1Rb;;(D(G)3j6%`3203Q6ehkqgQF|A$8qy8tDOA z?(^Oaxe*3Fu63#=&Lgkolb$?M-*r6ffIy}d8Z~{JoGfoqf0JA=F{;pw=RXmJKDp!UA_uor}fTJsF{p_A0s$&$NDJ z%>|!Ldm)VyL00NDmX%~{wc1aRB_>aJm1ju@2Ls4JCT6tCS;`_wxSZt7WyXp>xyish z*jt)%{Q*qMm&ySav>1N2Bx=iPB{r54O&A7lhMMSm`1jaKK)@Wd~N4C&^h&=Zi>ikd6?z z1{5D!#w$y!KTOAU<9!3ICJ&gP;sw*Yq&FYlQ_f%J+=)^Lrkl$zGzB3h+3jS;5Uqfe zQ%H?t@3O&YF`Osk6v|>4%PohoP!_Y4I-ufk3t;~>qinF;m?2v9Uxk|tdk^-{i6jn)30y%41xyc4Z zL?HstF!B$B%2)#aI+Of>tgRtr#@pEdOY#X}8Iw*#R-GG6N-W@z!=(w?zTqB?NJ~iB z3q3Ult{rtO?%u<_yM|s0RY+p(!_^c?bx#q7#~VhS6vOrrNaZk&+RI z*ex(VU^17r#Ab6!hzP8u2Bxjql~r3_oru4yYLS>1N-RyMVuF?q2RA7N$h~_^}gxpJDK%UQtp?c5&a+8pINr7_G#H)bQ$hWsO=16o%0>hP#}s{Dh4-w za%teaIfRD_{GIFV>7B5i-eA1IRA6|`b5S~LT^gx481z@Ad!L?7Q!+?Ru$jMm2j|{i zmqStD02|fB{0H8RIWd7!3ER{tC%{x#2SR5tHlw{uRen>FCuE|`Tw4N7Zju3J3O1*h zJI|=0@ED<_UPJSuExq)rg4vh*|K>Ee0=l? z09T14j6i9B(;H2}!so|)d7suk7B25o(>st{Aq^$}3J(VBb<&bMh3oPxW}acJMrWJ) zm(N3Eadu?`0UE)^P#`7k$N0Va|8vNs^-W8)WT2Bq`@F|TlSB(wt?Ro&OZKO-@{gx$ zoOJ;uYq^ z`_TPhCA;ccA#yj}o2JXUbn(AUFN@NZznHHnxttcrW@b+0ymES+Yo1ol5b+w#q8r<` zE;a+)PdESdY}2Hd!0*_h#sMe>FoWY4&lEzFC6HUH6eE4NyRh=T8l% zrG+cRTwm;Mm;Uid(}2Q4V6OYJKQ^tb5X!x^TOle7h&}K#a?+t0s&)tt9SQu2d#K7M zhe_UODTFGB_@lv{AnG5kgOYh(aV;pGR}*)w>UI@{>mq@e%0yC<`n)D2murKnkd|bt z-8+PVhIy9|T%0`09du*30jz3{1z152Tqz~!JZ)%8FoTnf&FVbO-IGfSv!0cxvkET- zaA!@2J{3Dxhwb$TH+5RX<9yXkXU@TppL26Y3J#teU>;&uNC^vHS{M|fIIS{D6S|z6 z{$;f&vrT=yQPQB@2tgn;gxjKgj85@Q6Z%xeX2qx#=#(Ro8J-9@Wk zbu4#^8qU&WhGkk~wLcaa4ADG)szaYOA!u8GQi()CY~r9P$i+jW0rC^vvl%^UR*4*K zXbCS41n_5M!HW_h@66xNQZpqJW~IpT>LqyTtd5d@3Yly^(T4iltGE1kn$AWGDhJq# zc7cvA#w3qYwp-xbF)dYI(Yj@=5Y|S+q+r=!Z$=%lJhN)IKXC->vta=86oOK{`F+<* z#xaB%`r}+io6*aiLzO=BfVC^Qu@{rcuprLG=Qvdqw&FTa1zL|^19llRCAb zrDT|}4YiXa6`o+cD+!>`vV!tTM=cyfDxhKz=dvQ7JI}onn2Xbn=M71Re5oBNnMeJFtOMawSyDhct(TbT=cS4z(9Pe)4GfzKY(^wHIA12E#r$7l?~Npb%u# z6O^4J07z&OQj|E5kaPDE&X1RZlQIYF?{-cbS(`vk4FaP1&eeHMN=s#39++M)p3Y~s zLdSDiXl*eqOEKd}pEH>k^8h!4*Mm=OiZD^&$D%+2AehZdfySU4RtGORv{~h@{lupw zx6^|K0O7CkWH)pOzGt)|tBz+(wxqP(q-0g5v*hi!}T-AzSy<51h!ZnrINSGitjcjbz zoC*wju=dX~>M72n8&DWO)P(m-_m?kY)2pI)v!DwkoHAT?wPZ)$)TFjW&<U;s2}Ni*6sn1%Cz&92lySAd7h80B0n{1S$_beb94C;qk(I~k^?Gymhq ztLv?&UKKR6F(EyR%$Wpm4eBSRlMGp(?2CAh8e)L){T8MJ)YOSA%F_YmjOV`W+;wA^ zhOXeQTK^!M4hLEqc!%&lDNLsyAk@pWElICd=;wX@u=M)cnv1gPH>}WY-V==v%aPO(;Ooqv| z_!o7__r%N)9EF*R51?v;f=SDo&am7N{A z8<(>&K-V??^4*a(r1MQQyGAv}1K2{wj+$Ms{%lhwnd7W`<{&xh{xyYAp7=t1?Ka|Z})cg;AJ`mZR%i#8W17g z4bn^-P_RqsEW+}J*H6|yCZm3Hd^6||l22=sX$GMk=x5VQ#!$HlilSiD2i$Ff+tsTn zVEtq$$%vnbM#+8jKDkLnw@XxA&ZBH_HBJC^GSpl{rIpoJPsAOwz_^zCD7l)YsWu>J z@1fMu>}oL*17r~-8I9hH(ONn*U;pEHH{Skfcl*`e;Y$;O82Eu%mofG#Yw=23v`EaR zJbynBn>H29_dTS4khz+Do;gqh8N@`;8*!C^E60-szl-m3sX%n9)F>u>JS&YtcT+k9 zM_W&-!r7!Zsy8ZvK4zVyDM98&EAW#fi&^V2YD zX4>8(UB6d}*$3G<81*i1pFeZwX|?w}x8@XDSH(5?pjr%3k2<;Qchs=@%ok#FogUg# zmm$(U#Yi<{bT~IT|Eu=dW-}^krb!TGO|Lrq1_tgFeq8s)QX2D+o0cgY-~pYJ7A&*# zd98Tdkei+)VAe1ie~|Z61z74rYC#dXNgj$qP)24$dR<6vN#|R3P9cReu+`|72*I!v z_<1XMnEn8YBM$F>h)#E8voYGJlV~m&@N7ECOm+nGZe(#jIS^*z%fT$0+jq0J0K+d? z4UC}(tDS}gD^fvb{+dmw?kp*+BsD|L;5%=TpbxS3`DS!&{&}PSDHp_3^{EO9tCk7j zd_6n^MwV;iubVgo&j);bwd0c(zlu&_s7M-`pz0W)7DNXRR>=Ve=*ucw{eYuDcAVn2 zM~S!v##h>Kbs5OE1}gV~^x|k>PXnz29)A90hUgP$xFuRXhq|YntH@GRNhX9)58U*W zZK~*uR7W-k)dc)qFfdX{(HZ~6zWAHhWl#_+`iA(0`bp~5bUXM>ZQ&KQ$nH(0RqN|b z`(kT^mW=Z^d$%j}*5QP@o~1{yfEZqh=X&+7hWEmmNu~bz-g#G3Z4Mm;(T)cX3N)1Y zPUtX|zdOjGgRV(SZ)z*h)u=ucb`X*B9V>Ly3B{`VF(`sgDKKF|-p;SU@UCT9dLZja zFYno^b*$>zij9%_Y7w5HF9V~petx#;anC(>M+H|Hng)5|2ZyCg4AkV7IC46;7=rN= zc_l#)o5&u3C1VvV8R%h{$t)|YSVIji)QqXOH5$0CVFC$p!@%$(OTi0iGOHftVtj>j@Zi zn1{h?r5=X?J%93WQ7igoF~1%U?IK*qw7N7zt>vdUBF`tEof?tLCSx)|Z{ks$&kZ-}DE0x#J#9->s_JxOQl z=jQ?Ekc$5HK4xmN2YBI0>6j!T17lhq^SFR9-E@AKc+Tnv{fxFfXLM<4 z$2)>@m#hE{_gqlAb&o37zswKa4+W?1x_e#l_NH5zQdA`q{&vNFRz?0+{+3I6?Re85 z6en8nBwVOOO0sMee-;7zELxHScAfBG5hFP8LIzr5y~V_aKS974pMS$Ocfj9jZ9`$c)uh!nm6nmulOz-A1RB&Ei4tg!mD6fxpjy9t= znF=1m6;2@iS$YG)tuH}6P183iJ))VWBd?a0M{c={R3?xx6LuH!4{$h_Dh)~aC~HX( z=30}@z~zr2tft&_Irgsu*P^L}*PBs0P$f2>X!yMw6tJYAhWpV5BZ@$xB0=F5TFx zXeMe!$7wpG*vNEzHGjYAo5WX8h!J3Q zo(zV0ofN(#`jGDxhJCqnDC4PGC0sEpI|KTBa%a_8kAmz49xnbz4-=0HQGHZD;k9?Z zs!CI_ddSQv`z~t(JYkPTtb_CHjE3DjFPDm>SIBY_hm<-Xnh5N|__^~itpgdPYFm&< zssX{BnfYO>36^2=w)Fb|YtELkRmvr5sX>)660y3$S|89WOP1V7fNgKY!z z@e_BxB_*~No(;YUudB4>o9U)X+uSHtrL}1StI|V~0H1$K;9cQ3#?~357;g&4@VhPr z-xkr=(DKU2tJU3H)smfyFK3?uXmiP}Ds|7QvQyO@w5slxyGzaxO5HAnvu?QZSyxs* zTPdVxL4Gf|SGwAtKN{A{Q7UR@#zP`-CSeoa_ z^)pruYSg_rStAU}s1;Fc(qE(iUa~f&F&>Q|eeX0C8IQGI z&RVUPb5!dU8L9PU_@|=A8nY3=l>)ZHU@4Y7S+06u$+JeD+0q%e>-y(5t5S>VtQlL$ zBpNQm;RIE%axj1Vmqj@|&dEV7t);u83L(i5YJ}WJx3dZRE~v#DX}rf;8bP0KGQ-LX zKaivu%GfHVHcT%)Rw46rR!DbSY*2NCt7c)xT+7id@;~?@7~|7JEq%~F1JxTIuBWK( zDct^-!z6nzUe*flrRsdvm%7Y2Y>#Khr3bm%)~|dY=9R86-cHN9-E?wlH}lp(CW%Pz zL#t@nKaUF_vjk>@pQ@4R8aALIZMGU;aQ>yq*JtjYHww2UM?=sI<$n=>q{cq4aDnD7)YhZQW99f)%3PM+`ZU={l2bDd3Y zWmKc8QM3MkTyNm@wu47gWc@Ibv*1~n$E$P`Sp@spL041m!yef)H41q z>m~(8*{zUq^F?6n2{>}rWz^qe#`4Kj*6+rI`IE^IoZ_ZKac)~aV)u6L#jc z=s*%N-+5elsC!mW<|UWfUQ%ENIP?2S`G-6w@9%a1{1w7|zKW)cSt_G?O_#ne?{LVD zW>!$`13Y&&EzkLZ-^6t8ivfCI%%8Prd^rPWig(@$S@8jY3z`giz&l(B1= zGB!@Jl;dQ!Hcal*S+@3Jk&NeqUy$ovn~a56l&+0$E~gu<>HSjQSQkIvZfxMc;>Y^8 zv+?vvM?RzHo%L_Oeez$?`fmY%1#l1I^Z)g~^G8U$$2&i-?hBeY&eGLAEE9G)NN1bT z))W*mtDV;RqZMxUVlYg%#o?HNK=nGkgOZf;PpqRn8(aViWf7+gyxHGBkR=r!tvp)c zl*4#sJX*n|#dLj)$1)x9!KA+!rYkFXM&r0AHfb`7Ay+py!g414SRiAJXu4wMKUT=x zlBqBOeUn*>5E%7JM*8FD;>qF7i|NKbwtI!Wnk$WE2Trn74y+Bg4<~ASGQJ#Kg$JZ# z*CGaRG*RQ62$GZbckiZx;APdjgHS8JCr5b!&krWMe*%=A#V9Tu$T`7w$bv_5!O)^ zcPl=%HsrtUOYAM~1F2VN%XqJ@4`G@+_v2(V z=o!yWp(49{#64lOl1Unqn5D?5LMHP>2<+)h?q4w5^GvnS?LW0qY~lDb@U=MsQa*v zljqiIwf#xbkCN#5i}tq>5g7)UV1A;*w=a%2RMKBTVW*Q!@aYgy-h#@pdEp8SEr?^T zE&sTQTu{>fb`xvKi=|+?G%aUuwwR(zuIky)UjdjP?f5KBvI#g(c_}C4=~s>G4Vw{D zuS#l|Ue*Xayj)|MQhLeIO0XGyq#1qE>VS~bH${jCCR+DX>3YK&g@Go9;onv)#TvW> znT%^8ft?3slju(pSuPMm1mKI;T-y>O87s{NHN=Hn7n#+xnq8uckW4eF?ToPyw-A#P zsK&&*k7s>TUAZl~Ydw`h#G=Ge5Df3Ej(wU@E#^py7?QDwTcPlKwWfnsWNNBm7(QtABst2GU;1+LD<>+x_QI$rthTmH;)&EUm zb}$8@#r2rtw#-nif4B)ry0gnTyB+-U3-qPr-jpKIlX==9t!n#ebg{2AtLj~7A~m^; zB|7R0shR6)&C=cgFfbYFM>RYook)EMLEUb|sB{(t@@MgIaGlm+2P%!245#mtTGPN? z;{yUS#IOz}A$rs*KCU=u-jTk67gK)v;}K<_s0q?jGfLJ0EyAGE8sF+^zh;fK)M*wU z0eeJFYm(`7cpoZ-!u`(VlXfS655uQ;>bgf`O0r7<+)rkV}>ud)LH? zI5gmHx#Ld=Cr>|)u~vl$B`M*=c6cd0uaVn5HVnY49+#lIesA*F(&XLv1OE^rs zlvX=A=a;ywl+8xzb)`soqxbP?ZP-HE)f=`(BN73urAuI0Pl*Vcl{K~V8h+#Jsc-4y z@VR&Df)XRpy|a~-W-V!fb79?^0tNr_4YpsgaGPFIZG^{i|2G8!m{dv`&>AZslOvZX z35CwlOM$MgbL-aOW$8_HnorT85uCDMqHW;l@u)7Q!}QGP6ovmf-+Uzb-@oycKpl_C z*M$5Or6>kRvNpuKclF(7bl8kuHQv1gr32pdEm#)GU1{{<_(_NF4mv-P7(tFR9WFBX zdlgYEs$b0}i|K0n+lJ`&jBhL@q0W2nffxeAeE{Og(R9w7g%;jL=cE;#;Fe@EaU8Vb zNz-uo%%ions3WG|Bs`bbz1cua30lZ13ovC`0*z*@4|@VU(Al&P;=;Rkk^-UhEPj0) z3Go+x6AfDm<}BXP`DASA4@UI{1b01YMwd1CDi-p5vSV$AxS<8^e$;X76AyTu^H7Q? z8K1gFZVpqI!bia027Fgk#gl|m$_q@2CYPcmf9o1Pkc*`1D=1Rd`HPoA@SCRqRY0o0 zWAK)i>0n!PLuz}{X&8e9#arYKnmW2W7l6JW4`BWb!I=0IsAgGjx7I}qeOT`zsQXvZ zE3Sx_TW~;hSbK3SfP4e|aRyic4dlU}5qWDxFD660^Pwc39F8{n>&$bQRCN$$<|+MS z(wCwueRBqOIx(so@FEr9C1)6Uqi97bYbm&s#Pg>E#%$ZyCdb5$>14)S8@kR>t-7$SRaQ z8}#_zc4onc@PmRV;S&Kh~KuVBo$X|ygHm9IDr z8Xp``Xk%$l&BMem@T@#?^)pIGSNG`b8u{g92$pa$>H0IwGKw@Bty-GI&ct0^Q1Xt; z1T_|C<>C@Jck_yqms*sF%LPGb>kZ6!px+RWT;0+mQxwSme60N*kPCP}HXqqa zB46!Cvfe^<*5qO^9L(=yDymn9nQlHR)&@LsY>+ilrwRa({V^gfuDO{?QIO;iQown6 z{H9ei2i8uwqe>$-4DJuoZ8uEFv3gf(w&@ukgt#~4T3B8+xV1VOp8O?5tEP{L#MkM4 z*Q1too!+#TrczfOr3YxCY~|CVUqSoy$Rh&E>E_O2iupA}L3?lFOmv^J*+MmW^aTWK z<4ie|sR%TUDUqp%z;MHlG;-J$e4O}?R1m3Ok9$9p^!yU1(speS;(Ui*3=?B8>~T>$!tHl&4GE z;=rKm5X1nOoQ7pk4Ri>DTkD`9jsx7TO-cRGx0{u0mA`p#3-jjTsS4Il(2rOMhw*4J zRBTPleBS(DMf@Fdk}NlgKi=i>0797L6L{p<0rJDI>UKYJ>!aHKCX#ysCAM;aF*LnpcfHAQNHwamR&vpk zzIUgbagQ276dr14Q{~C09H8?{=h9lL3A-4KXuR`&bYME z?PNCWQwAAo>Tf3g8oWRhtVsppl2U|Ga8Hs*{FqqMc`PI%mxXB z5gF?zC&t)n^Z@r%aIlR{_vX5xPse84Iu|4r?V1BFn|1L4nK)uw0o zrwR<1;u;s@+?;Whjz$IJICuwBgwu2Eb8;>6j?GeOPJ)1yS+*4- zN84M^F|3cATn1y~i%uroiEBekPipd&dln;(LV~ zo{?*aoG|%^Xx5X*SfS3?PLI7rz-k-eqijY7Fj-xytow!uRiT%~S?WQ3E*G9HMs+`g zd^~#y=ANdcThn1<^Yu+zrBBA0_-dB)i;S)Iv8YSk@kjOz><>n@&8RK7K3*9;SpU?p zRs6H`N?-x4gn^g4M{@bF3Lkd&_Mw)Ppg*r668r3Fz3HNETS*>9>%>SNdS{MY*jO^% zvvf4Mkw=|&)PM{59zL7`+<85iD)}Ei3OFDdU^&eET94MWg&0Jx<6Q42O)+^2dDl)!-%_lM78w_@xVl zHihUL{(Q6=wHx4QMt?S<$NXEq4(f%K4P&4YCNbBSNS_P6=4At$%fC{dC<%deb{v%Q z0A9DeXo{x36s<5D&*R`n^yWj%eoh^~De1v5%A3>NnrbNdqyR4LjpG2sdG10YXX}>+ zO=w-B8wnnD=llBiOgEcl_qu2Tv_QF(t^}Xvh4WV>*-6IIe>_j$C$s(@rC`WU0Y5;d zCy6p>15Sbo9}6i9XV@x2&FC-PG7K*qG=Nw3 zuwE3^B@KP(;)zzCVcFEa;GBvN|3fBMc5I91m{Wm=Zm*xFe9o@ghd~V%I3?jx%rj$* z6Fg3WU*SxhlhJfAOzVAei6k?mvdKA;UWWR`n0rY>U4xiN(DXt{=(NwaDs*WO59*SW zb;YV?8I#Z~<`Vd|mpbbWiuj0T86#0O3`nxEUquFRqg$xn3Fg*FaB;xbQ4&}&@-4Xc zLQas2bU3+{y)hm^uz_L?L+Mu51d}S-3g*#GI=hc9lg#0u0N4;|L6DAq?D_%Qi2hr( z`HK&<4HqKr&;gL*;I?8>d?s)@G!+-0N#w2JQyR*bo%_|O?q^2)>(@JaQOB4UX0_Dj zSd(Bw_5AxUnt@%M6;@0l|PoOKqxvO`6thQ^_SVY5;6G>I^|MVf@{~!U} zv%jpaMrXLr&5$wmTu>9!)S}dj^;)P?tx+r|)y)w##k*>v#3IzebGkSD-eQ=D6Ve|D zvX_CeMH1nyegfGOBhy}{287gRnBF#l&;<~2>>*h{k>`7fvIQic!;)x52tITasCALf z;J~96G8f}s9>2-sC-%manhx|HoKzwL?bu|IMbgKb3=A2lW+Snt#f)dUKRMZ2)$WJf z`hY=4$?!J0M{iS@4wMQMb9sHdiUOv90~y@17R~``_ku9p&s1s?mP9cbj6Sp&2)?X8 z%X0un^@|CFj^MPUt{b|bLln$pBsVKdMk!CCrGPPiFJPJ8C2-T=f{}c_Pi|5(8cfp| zb))urdtImF)?3PJ!G{|F{p|*3=bm83%KC8?{S;-6VZB9A5Uo0#A{m-YybSuUsT%0=Ub<{D%azW@B*Ghth$rW;Aj6x zV4T(t{x;glgi3;Q$flDq0~`|t_BRFj5|f1gw&B06R!jW1y}tfEJX+PB@gE4yyj|z1 zzg=LRe1jK!(;oi`{j;GxdqV9t^mfm_gKtXccO>+y!(0E+VQ>6Y1p1ONk2*ONHopHZ z!hfFeBk|AEcDpLzmx6n>?>=p7&7O9!nfUiv{%qrk{B92U_EM0y0Pex}9P+CQ{yCEC zJw*71!T)a9_?JUe{oy+X#-ra5@;cxp2!{~7l06mkzR<4mI*@iqNe@?9!}usokW|=` z_G2HJtSTQ%>!&+8Zqi>I!>0PNSw(#biZzJ+g53Pk7%oFz9kx@qB-RA%4|TZIYTG`hKNvmbMwpWS0A^H@5{CL-0lUE6x>fxOwX=|Ujj592`f z;UCSl(LR7wol-zas*SJf=bU*v9{Za`e1=WqD71MAfX@%`m7sF#T(k2v4jGo2!^%Gmu4N z9M$||Y3^xyMlBetR#M@}CSQSsyDPo|drAA-Ztk=v{{N@lt(|zQ{cS;wh|)6x+$bFm zFNw{4&9$?E6iVV#xUc9hTVYrvItm7dzpNVEhm-X& z-&NRIn~lUfV^|1vo7YM(drytZ~3$e&FN%qE9~Vp z?KLWF@&o2tV2&)DZK)};_;9v@Q6L^giJt|w8l08x`uVa$Ru&@k^IR11zySha%}Qq! z>1#YH;Qpz$B84enew=wSrzCl~$|=ys_71va-58pma`J>qM8|6#FC4|rV8f;x(Bw0* zU|UTYpz8(-TI)%V6p>&TEWCXr=~jw1zZ7T&xI5CF6W-$!0qsV%yKktw{gzkld2F{v z|BA{Uv#hgxna(3TgUASgd2^uo(S}@So4ElEZWhB$9@N3&f53Dpi!xpP{YejEBB48& zL;KNWta+)+-4^q3mT5*U`N@}5TDaUqfeUC*rR%E?9DzOD-3ThcV2leb|A9;J;#<5rnu`RuAq zt86|_>JauHffQ9>x{621bZU4as}NJxs?u~lu!gIYy#>f zo#YKCR|Ad^&A?3r;vsadYiBV-iygm1ClVS(bp^c%IOD;A&!|xQETwux+53TJrI=ij zQFZl4Y1UXMLXtnqYX1n+dN4A3 zRd}T>0D0CQM@;kDfY%)1^JK%9#Z^<}dS(mbo@h zk<*`~WF|z%W={SKsyH&(3dSpY4`nFy0R)5Gn@k6&e1HfM!G>N<1#eYPTY{Us5)9iN zMG*JqF}ajqP3IqU*EYdpX>0eZ`b8TL`WldJjS&JE%xyO$M{7`OK}57_*)DIn7^Oq1 z(qc9CY1xeVK>0mmezJ&E+kZT@uVOp+ra1sU_9#246BXAsdNj(?w0{+$WXgr=NqUj) z{kh>o(|(e;*1way%u-r&YOL!NOI7ut@0Q>6Bd#n2G|{~GVSCvTK>D4F_k_jz7~_;A zyQKe5Fq4sHB?Iw_z$gtrLXd#3#FqkzRaAhoQ>oLZ+1MG6@`KscaNIFtVKW-s)hXrM zY&7(w1FEJS_T(jnl6LtgydUu6tOq~O@^zlWvg?&tc*>AbvPj!KPDe)dI}Q{a!bHLR zk8m%KKVS95d+m8T211rU#9KVV(J2<8hXdT8|! z@{A|T;H5RQ7JrovL5_#5#W?%0NYh`S++?E}tv77+KA|!UNpCjEGUT`r)xF_-9#vIY zM$5W2QCG=rH89dyVe++EI-TW;*KDpQrUFINWzFQbyB|GF_@|oQ=^NvN+DX*Hv-bKF<1lBW_La5U1;Yi&)%2{b z6mZH$OVmWm;8-!rJ-!Ub$I1$v;UL-HDBg(D=iR5In+`6Gio zIQ|<&1ksKe5i$P>7y2E;g{;yvMJ9$K7o-AbB6o|LUVW&~>`+i!g5cV;lYKH2yZNtH z!DSB^yS|6Mn#mfwr0RC)e?GFSEBk7H#4497w5S&|eB|`&N&X6bZM791UUPKYrwU{X zf1VQp8ID@z`V+qT5BO)c)=$%EmZsP7Xg*!K=ED;8AD(vB zzccGUw7=~<{ZIXef4}+l+oQ754&XmB9?OCbvt*os zUx{ESRY?PJqQ&7Fr~PfdA@tn%lyHioefnu{GD~nFGOIo%mx=LXGKZJhT`x!QRy>j) zDJ%^U7IMj(rM&cjDn(?KLdZ)LVLc%(vBXyf`wvwsZlFdoQ-90MWEsz%nt_%@6<0z) z36Ne4(`ui>f5u|lUkvGmbF|usCH-_RQX@4L@RWfB}>eRQ?1$Wc2 zCceXO-gb5bB@mO|*XldvFAoNHf<;J=F1e_t+h8KamUuq9M*zf^(`Te?6l~q?wYFOC z=>k>nbQ+HaS*Aj_5KNg?A|{)rJ$^Ol-;qZz!<-?SL!4Q4{A^yx;}lOVDo64%0-GK& z)RaPV$9a~;Y!ch_iC)cz3UWSS%`b}471OFDk*R74g?SuI?=PTGT}hnk9Na?UrkM3_rQR7N%8w5@;9 zl)UXM1YhvHe7T|0_rNXu6T;)iEeaF%3>Q6)I!)gZL!E+EpCnJsrILG+@~!b}>bw)y4u7*0 zMqSUdP}qZ{^g$gqsXzw(ELfXoI>}J~RnTH4_I|R+;)0vR{v=1aSE>4gvj#pZo2OGP zzcD6ODlZJg6yEr%pP+w*W?mE&urXvHm8*vwP6>4apCF1IICy;=Se81%w-5wUS1>e9 zx9E%3g#R}>H^PJFM7z|^!uPLP7#hC z!RQUC00WSZQAo{4WAEZ-lue`+nbX^f-J=PoN$)*~c96@rHy_?ZAX1c1zGZrSz!fWG zxR?9M`0DE{QgSUUNl)~^%(4Y@`)Yd_3tfqOeq4=#M%!FjV!c8JxDuZ`qHy=wJeQqw zP6?P71_UW|JgF350D5WmL%4$o1)y=de&^HM7SEZ~3Pg2wcu$tfDK-AZ)`er*| zsWFevM~@qnU@65S8=K{_kl*)T=PRPi+DF>6Pg3ReQ4Id6R_gXIW5593D*ns=!O%W6 zmus$kqj5rnaD?z+p&z%Y&5ZT7PSF^ z=F}X@eB`6mXDRr((4d0&%ATP}O>|wC8EUmN-NIshZY~X-_qWT_t9Nd&1z+QN?{LmO zi5Fmb9cy1veZ`m5d;ga-rh!H&{u-`pnAjgw-C=3`I{ZDkLOG7@)h=ad*#K4{aj;gD z-d%QR+gJGjL|Nrd0`*Pz(k}te}Gs5qHWykD*ZV)ZCFXNz1$EdG%xZCdS zZhQ8uf*%Btkc1I$CP=9@KHm8LIuCH-d%}H^uQIc$P?szKRLdUEtQ`~6B2bm6YgT4f z<}Wi-3sO5_zG8G4XIDI@kZKATUJoxN^WUtlQQ$v0QB+P&lqew%hUHedOOe9m!(%`- z6qIFk6&dpT0{puz!1k3UKF4U&eZFGT(&cR#dOUDbvr? zBDw9nc`zJfUNnvUvxt0Z4Sf~w&=UTgr_1tTx|p>YKP?ly1>8LxE%@!L*vK#)z8C(L zHDdb`XK%H*^DU39b)f_LxY0xqG*XW(#j;V$!SD1#$Wd>Ekp6P@G`hGE!&AGIyLNnt zpIQ}}31R2zH_m(4ceCCmqMX*sjgjwQ2&kPs$yw%nv)ZjaGW6K91X$lS(N*-@#eB_| zkolp|XUK*6_N7A{@D>?SS-0!3PAMU_*%=(MO7Rxj$6Ef|<758S$e;PHe9Y zPRO!{P1g)Wo7ppv=Ib@!OXymIhH8N-X(73ZYl(nRrm44Ljph#gdq&P#OFI8REL%69 zE)FS_T`tb4$vMbq-KaJ?i$3>52kI{(Qc~Mp(swIcN+EAA5wojgS^GN41RSC;;{g?v zGk;atl)c))8F){j?zXa=)sv)2RItAATM_eov^alhJMD8X0?hAOiya-;Y?pWWYoz^i zk14||_;=ZfT_+S=S#06m#4^NDD7)>G3Wt?FF>(JKmh^6o*M34Fr{D6&fy-U|;`N`0 zeqmy9KVDvCxvubI{r5q)8u~{D#>{N-WA1)!Vn@_JlYaZ?2pC-hjFucy@=A#*edK`{ zj_XSq{~jH}FQb%oM@ue+(pfh=me-e=Ndg*weaUx}D{B9Cd)Pd+3T26Ht9AFvdMiUf zZU?94c3kM`<$9axHH1ErV{=V0rDv<_rCD14W=qQjC(G*P$HaHhkv_I1J|XaJ^?6&_ zweAOzDs*-0xUPbRvKBEQfWP6O)(n!s?K$tXD!~i*mn3BS5r{%uRKZ9jGK<19OL!l6 zAQMz~q}dK0Fk4eg%0Rk;7Kl}Tw*G7s$?yF1Gj7jUpP8F;R!~Yh*JuXfGP>A0;1)u7 zRt(Y;Kl_`N(2#H75K7DCRd%*1e(XaHAjHU>5LZzycLj4=DS5er6L4&L_u!{|!Pub5 z8;Zisp|vDVW89qf3e?REt=txQ6pX2HA0%L#t{${zSTW2EIM(rg*PBvobYrw=bo6gM zINtDte7HHRmT%YkRL1%DL0R68ENx*>Yf88Nc;IXyy>MiNG~@uJ@C+Yny1nB9FHSkyk^`q^jxXYxPvKUe1n}M#P}dTtjN05sGwa2y~J_- zK7aN0-wCFp;@2fkr{Mn;Xb^E9L!|rfw<-hYhmlQ4P&o$b6ThexwWX@q(cUeU6u6sx z?CEoXv4P>{b&wADX%T}hj_fj@6B9+i>1$v#M_<2v`dTh-XV<}6uOr-iiZ$D=AX*Ci zOfa`n9>I}02firuWwF{EvSe)G`5!@&h?4ViT5cuFQ(hum6#}G8)6^Gu>Vrm3!+J~)gtYC8EpIqff=sSP?IsK5OHl3~zGjV>C^4Tg7pAK}6!@FhB7H!jc zK+D&Ccj=+744GsPd6M^{C!otL%4H#^C4VPYt4^1)T1R%`FS{}5vw;D|<3WfB=(MBg_3FB@dY}VXOoF>MxI7$&U*XZL^=i4o_3^Bim;>CE=tpCQ`o$82%F8+k{evbN0 zwc&KIK%$I~|5X3|bAPYFz)A{-lDhTafkB!(V&Jg*Ywg%s7Y=!@&_|3jmM%XP-@?(WH4ZzTc-tNz|>ALOQrvin9n;Zmu*9p;6w)eaJuZn;vb_<~kQ>9nVib_bF zxt%7_CPjiLFmWN66z5oT6@lxAoSXa_aPC^f%UY^Z?J7AfPg z2L9WNwKN*wCWf>==H&;8Hoh=uuFn`rI(}Vd0ynv9OIv||{P9X!w1KIku^@0#w2_iI+iGK>7bIR>qoQAhI;r$Y(ynWh^SW03zl>{0@Sc;9ahrZst>1@K` zo55~oR%c^b#9IUvW$=N8qC5+SKYSFuEv6Ssacs<^*9ExkCKz0{K)>9oi6ZiZsz!j6 zUVU(8uE6M}Oj^-dxvA7DZ<+fRj&Izt{;TQq#JNs4Q-amhm!5Wg^J1%kx!%Dt+rxq! zIC_z*^i z!}q2jXX`*pKh*{;4)TJKi5ML(2>SvTR*>yLahFa9mayCCde9KMW%Ue)v98KRxn&Je z#?8Ag*{86IFs1^*IAbPLdQC!ZVXKF@%UkjKVs2v-j_eQMf=ei}0nN7$3&=o>iU9qx z=7!Nvx`jXM71Ss6F=2K z#A%okSTS<*+DF(?#b0Iv=Lvhu{fEf&^@;xB!ICA!~2H%&?4lEaUf&!L=p0mBp5MmtnK zOp+OofTZGflmCLFA<1LgYoV8rX$!USjI8T1>!aeJfVtr@DDR zr~RbybsZzoZ-6?3W7Qn4KzHS?Gdip{Ta81~*1eX_$hUppm!1UHfBF5tzt4X){(JS{ ze7^VuBGx1xJmq(JAK>>+`0pS6>F|?>4*vVcpFaH4@A&V(8T|Jq}?gdH&+%C*J}D*Lt-Ahfacn8EQBf5U&eNYbogT2r(H_T3|?Sp<$EtBMAe4 zvo;{NkCrz*g&HqJ^z{vb>8&jwc^*~1*8%C>D`G;*Yh3&9pBz2kczP*Q?E?P(RPtyA*IH>|UHgl0(FwfczQx%xjrZn?k)1 zGqgtV7LOjmlVGjIA}y8}@JU@%RiDyB>|MBW&FiXkkuOg3?BeEhQ)+~Mg1mkSZ@$78 zN1`h)Wls|KC9d!{KrQJF*pFI+bEKB{a|rV7^?HA<7cd2zKNn0H_J#S?IMABGNGC8W zHp2}q1-Z|}lB{+M!cbCs?T=M-die02-dZ$Qh1_CN>BToRKp78hW5)(^^}t{PX@$Gj z!(80cX#qEhI?+ql%4J1i<8yKgY^bIcFED0MPC#6#m)H5`yb7V6TZ=i9gEj*8l2~9o zVg_VFt~z87?PVAwCyGyM%yIf67-(cn_{&Q!wRIpBnE=w@Ejz? zNPG(5WFDZdeUk?I=x|L15qh?p&uyk7k$NORJM62(*`vclCzYRv!fwCz0oX(N`a--~ zwN^}F6*Fdcv$4UZfk(&tw~RXt?xcC|H*zGJ{ze<82Ty1`i*gH%V_KRCdHXljT6+=R zUGwf{A=;9~*wVlLK257dK*}cKwvayYw?go>yKi8By599~*EDNpf1&0D#!rg_;ylPC ztCyEK;+K&}AOZhED3O6^J#ut;=By?}tZZH^&$k!yTr)}FC8PxW(77o+hg(3hXcDk>_jS}&g`hs%%|I3KF6;hSJxAes*hVX9=nFU&cG$7#?CQG z8E>dl291fm@74N1mig5StDA#_N#e^1RqE{fp7D&yY0*H`!=ZwKbgA4A$ptjSrh3gIV z3+<=Q>*Bo@LDy}1$4kUt6lzV7L~R+3L(9TQ%xSdULAYyLBj)35EhMI5=Dd~ieJwcG zYm2)pQLuQPGf20iS`Ln7TM`tEMnqpkhnxUxC2(&LfhE*%C1v5EmW3%7ks5JwNCqRM zFRk`-xOFYg!Qce$Xz0TcxjP)d#f6EDhN1n>Gw+=wv zQ=Yd9=<}1z8}cD~N7@5QiJF zoyarL>}B6L>i3g#WcO|>?s(O2C-rz_?n3x+n6E$hG?F^Ag~t;cWfoIdF$oPEo#zS8 zlQbDbz!xYtN4|k9TzV6h3kjtv_Tp(Fb|ZRJI<)Gohhc{@7eoXm^)jEMH|rLy$>?~i zuMlu7ZFDV1XqAITIY(VDJ#&|1EIE^Ev;FwG6VWDcvbmGVm+%jf?sKk+ae-PlJ4Y9`}Aj!1a)f%`1P7-SjQ<4S9 z6t|Wzr@L8>7bJ*PJ+fVE>SE|8nc?LNxDYUUX_}_SwBpQ222+J37ZipFuYDBF%W8X! z|7W{%Obn{pYM^fXEU%ly^wZ7ceGg^RQm(umTnNArM1gte8qZyKq8rF|vzww%62{S^ zTmcodca9OBdkUUg=r}7mR~mg&6y#GS*kwMw%+HG<#_vV>)e0`2a=0apE(8yhXm|#_ z^I}C>M&t*UnkRrgr@+UoSxujP|Ki2S+kp+TeI^!dvB9M&8tv88S14{>$=>+d z7-hLnEB5`ak#Sqd5-QeYHO;1mAsd5j(2XrNskl{^>-wkw(TLv9izV}QWkggg1SGt8 zke1tG5t|10AEAB)t;vM+Dn{s3h;K88V5pcm-{x|+GkxT6M*S$;sijT2P8JUv=EAMY zx57MDtzLGyjY*elqrTdrooljd1jxj_peAVX-Q}9O2XSv>np!b68TX{ud^WVv0^w@A zg2N#8{g662ymUJMx}SW;nWKSU19neij7={pgJ(V2p)K6MkC%Da(0Qs-FcDk4{D%~R z%VM|PgSDyv# z#n_7k`2pi*W`98rQTIW8_)GT!=pLOt;&>2U5y!^)67w@6hz^+0@j5{@e~46~)!7-m zL4NBHTdb~N0?5UN?mF~E)$9t4R2Yd4axKEkE%mjMy0S>yWHxV=gw0^g=1f@dIsYDU z;5CDrpye83E7s(=p$UOchP;Rr@9_ScA!pJ7PA94|T%IfDg}H{vnbDa&2D*j196?~<yYgJve+z(`EksA zHW^F;GGfy;_?;Q3_xC=KEX*F0!$F(ibRK=|1Axy?<(i-L9=sKM%ap*GOG_c0Evh`MOXXBgoLtZqgU)!C>=W+*aR@WeQ&w<~b11?TE ziw_1yE%ejF5`Y#GJ8crWkf1>2Vb9l9zO)=NjaD1PoW7XE@S>Zz|%>W#08e6~mZtB0z(k->TF>(>U{L9L)7@qxHg2X~n1w44gO)Yi2a^yQwiZ^PMmckVo{CTZ z<8Aip=CN3ULe{q1u(E9%TQQ?EO(>SW(`CPmc4F@>A^cJU70 zpPn6l>J{pMK549nRI0)%rKjwnYS}A%`!&6U(fGg(BM;=Sk9BmmLSVE)zbViw6zMIa zk!t)t2lCR3#x~HiBo={rJ!f9cDGpa#WiaTbHD+BAdu}YBi>;7BjZ8dqligq39VrR;2Nyic^hLQW@(n{S z`Y?K!MBHZp26{~1duERzn7zJ;lojhZ_qrekpoN3to$&R#9^D^j*X3+Wq1N^H?ng0E z9Y%+sY|xJ{#ScoIG;o?JFKsd+sV|No<-?7Gac3@# zqbIalZRCrA;d)Wy#6}f5u2oxd-^fKZWO0A2BHF96+~*MjkYOp@UM5U4VT;xAq?EI- zpQcKN`siVQ04atjIS!tPbeDv#3NK$AnbhJZZ;l2MO>&cCMvU!+DE7^hBMpxR9axKZ zCdRV{4yNE=YD>Do+)i;nMNV)$Je;^wT|>8y*<(ArxW^_Q$OjE8!+xh*Iu3y=+lz%pA$!FPUFVxwY!9+gnl4v7)U3qhIbu*rgGofR@0A91&{5Y9VXo9ig943xVQoX zFjjx~*tbp%>t{zTq>}>jVK*H_^sL_>bW}iCnZA>7nz9XVw1<|>AS<$9|AKR2qPa`R!&@&ah}4LMWH^{;3DBe`-YUqii@dr3 zB-^~W9=OA<{GYP!XSutDTsz~~H8IBCJXr!0 z0U>6VJHKR9)-#kqIJ-!lZ~(F6CCnV;ABdU!8iq3il}dQC50^rNDx_?StzT9L0VM=7 zQlXToGB=3d6wqUS=bl8C!LuX-&lQ-}5-i?u299v%qk~9ZZzD4M@G`;2Rdje?93(=x zNSU-JdSsmSF!Ga{plQvpDo`#NeNu~^p-U;qJ)6j4p1U-34S;@^uldSihID;H`coSM zsi^|c6g94WcD`i6R2PHOTKsKJ4-95L-C8<;nk5$Bc3>yfc&=2mf~JG!5Uy;+*4{=Z zCyL6+iMj?6`r=aO)gW~vwX#>>&8&(DmerNN$N!+&Wz;fuo_YI>o_*$ZCmIwIgYz&^ zb~2}gnw*KX;g|Wkx)3jZtny3+9ncwCIiNy+=gCy|Z+>V6x3bx5ljtKx)qY@hlVn0|!I6Cn zSah1K9SVoX6@6n(>!01IH1kYKP2&ZEn`;?j)~8)&+%mp(k+S=il%ihf6(+TX814xIgX)+qP@>}?DcR~6^M_Z6kdA3&&v$}znZ$6Fi=h}I&N+Ai z%L}sk>KYDu9H#3Ma|1~X3wjfs&sV4U+*6weNWmQMNZOO25xeE|0xV;*j=LcZu4`+x zv~3lKZF(l$x36lGZPMU5q-0RQ*g)ShG;kPjjxBCER2vZYR~eceG0ie8cFW1g=qBWQ zgwlXVD-C6i4OHiNTb}Ro^CFW;-9|dSo0|#IDcMwBnhGsW^NX3W1Kum`rfj=nNADI> zsA;;x)wy|@iWG^>%XK6f{{+wac;D%rpGS0~Wg3hI% zfwMAn&REm0O!lR|{>xg*F(AQCE19@CyZtV<$QHd#(ZjS5`~Y)6jK3Z}VeFf>_GrbJ z;E)hY2vCqlA2{&h*hefBGlq0G2C+Tkva{#>>z-uK^6F-yL~qF&gi z>V#v>aP|{!kr8+MD9Y)3mo96Mp&=c_DV)_MLyz4{Q}e6G6Xxw=zUE65?=a*C-PYY3 z9p4gWEw+F?OneINoNNnNDxNl8zRcH&>EgiIb?v5l1Oxp|L6TX{;0uF>3SQx~kfN4* zH3?bQrrwG$8aMdGZA3H;E@|Fa^xhY0#uxXz>bk9+N~NjV$6VFbN>#hJXIt`bA8}6a zL7UaDYUlFiJZLg9tVFKkh;0J08}l19<0wexrj_rs&*`xBz2Bm7;$HL|)zTrUrQo>< zpt3^vz9^RPm&Ab}Xa>SVar3 zH|4f4LoPt1RQIO(smf&oIpTF(gZOvUkj$PEzQd{IsE?LMhO1kmv!D0$f_*Ce^StMV zCSkrak5@l4J%+woO`9}yPJsgkNcykgNH#u)2QxonzORE*4`<;69w5tUBY3#_C4G|2 zc#HzRL{2yPFQ8rN0}q_tZQ-ar1AV*X?Bt|xdR? z?mf)qT1fk9gmBx8%@Jct&p-(?O6%WjX*ry-tRCDCJe(;h@{VndLns`87$9GAS3hKm z#9o7HBSr@PA(u1w=*+T37|Rz!H`Rg~={1K+&(?Hq0jV4S5HQeBFJ<|$pMJK^))y~_ zpQIlRpUv~?LdX)Yo<_CBql#9GlA=3a!yEL=fq)H@!pVuhI!;a~N7%DhpAEOG;iG4V zpVGm-&Sz+jUT7OX?a~iSbmtsUf>n;ugAY8tTA3B+dA3}w3(wg1+qQi4DYoG3ULdZGPe*Uo)c-r!zLqWZ1L(3>AnFRHrGJZr6ny$+ZElBq~56FK|n4U z(Be4IVg_P~J4phy>oWwH9oGc3afOXHG^p(+oN&2-U52f|bheWvDrj}|j2L`9WoXXi z&^B1vTywMPNRY1RKusFN8E_c3*kG`#X0xJ62D?4h^d{|9Q!8jb#%IpvA2d?2+6q|< zWBvI-aO?MY36p%5+9I`MgO!o;;H_MPB<~6m=0kFd$y6UyWb|*@0Q2ONZDID-12o-Ye-;HDEGPWm$yeuiAd2yakZwRAtC=N>FM!SkGi#4uw zI62dX!Lwri|A9u$JA>b2RPo9IK0-(Zrka(YQZW-7C8D(9j6^VX>YjpaXFSHlZFPe)+FS9JGOv+kgm zi2uh|GR}4RLAHN5GGcX0u^|N<6u8f`(x& zE2X4`3@la{iB$B&=&`EZsBL9AJ694ZXT~NT>#u+Wd$TS^QF*>xZHnW(sw6o6g#XUo zd%Ag&1LMP6X6|f*HZ3nnJkWi2r4evcI?bzX9PYn@jyg`$G{KOYNz!eR24d~Vhhz$F zLZT7&N)9)J)_C}R+i}LsK|?9E+Q#aShO&B*j_c(bE=~5QOb&n>3g2PqKzO6dqvl?= zal!^>(86c&CemK_x8Fc_$dB8+(i|=NM;w%tdM>s6x~t5L#)mo5P~a zuKf_LMJY}ka5YDOPos~GfC2KN%bMN~euoYI74Ux%gYv=Y;lp=#g7>>0{_o)@pL}xQ z;Qt;zJpAZ){NLXU{_mZ@|MvR*{?U|TEh>GJwp!CYP;_zv>+j@5B^8ALP7UTu|IEu% z?*Zbcc0>~XDi`ODg(*xG0k#~YGztxqf1RY8g{^_1#?kgaP_M8{k4NWU~N zbB~rcN%R7N9p*h7#yAXE-0S^We=!zB|5hx=luesrN*;*;`%AuA?AG5e%PsMK?RbN~ zE{mIy?sku)rNvsz;;7b=k0%r!JQ3eh9#k+D*$h@hh6$VTf)1~9;!QCvLAQ{dIhKC& zaP0&(@%}`6{Pbiz@9Uyk&94fHg(qmjP1&@pz}*Ql!)(f#SO9Lee=|%3&8ka9lgO40LN!7&-AQZ&QJo4`!KRXRWv>#hdrY@ zLeD;>SRaX2lG(0Q^)gdlbUnjwMz>+)o~kS0k;+&Nhw(wWP*?$^1J_9OYDpgr|^* za%NmSy6-E5ddwE{yrB^693cyW4Y^bK83Gc>sQzH7A1j~6d9hfec(){>xb+=NpeWqN zph;lH7XYUN%HGvR>Uz~w1bpk##l_TkaW8uN-IMP|(d%L%b`Wi2+yuotz(b-iK5W}i z=yrem+q{U5UVR;{cc=4mI=@MsFJ!m~;UHO7<+KWP2QS~|h`-C1Aiv+Nq1a+o)M)N} zM=!)C+lW(b85j)(teu|EmNLIyxXk4&SAo{yLZFHP7T>8j0>h=&^P=%(9WK^f4B)0@ zi9zpY$%a6~gQ78V8!(<4d>nVOVA2}@24`9x*uc}!*Z^h~q zMTe7sYpcyBoa4;cC2<$}b_)aB&HoCwZQt{IEGO{3%x&`b`!LHkyzg2pqQ*R0E<%JTJFf2Zxd@IHU1-)#DjaUR{lm(0`6`&7-qZ~Z0rAn+H1aR|KUGB=MC z?>pX6@_rxlPbTk+`dGU4m+ZsM`v8hVnV!o#SPdj#c^?wO%^eU25y>O%9M12@7u=KH z`&xa%-}*zGwBUWmkP_biL#D{#Zy!U{c>hmXf{VZ2=65h`AoXyRYiZ|BzF`PY?AJhO zx|IO?$wvkY_hE+DIz?i?}N$UqS$T9YAX%}Q2M~aluFA6 z3>YYshd&z+WJF^O2>A#_imD@eP>bq;(IfaNRRMCu_cSQgHm>IZ-;E4RWc(sr_BiD% zvjfG`i(_CbL{X|U@a2N(2E2$R?qC(K=lOC3b|jobzT90 zq^rusZobWz#cJoIvHvi}>%J71JH^MPn7>0VKk-`yc2#aPz8X0r@k*U{#H30IL2BBG zN&!NT@NedONLN|=ARi<|Iiru^Jil4(wz2H?h^&^<;b)0!_HBG6T~VD}SD-K`~`2P@*ByS4CPxC%~+ zR|lUu%Yc?_Z2(pS>zI=y9~eh*EpykF$R%}sF#o#19j$#?gDZG zYu=VZj)~OOc_m2GC9yuL#NeytB7K

K!fOVg6<{(cf0a%U*C&SW-Rz2-v55`Q2g`z{qH~B{qG}MTgVEZ3#oke zq!9Acw?Z4J)qTA8l^43}8=c`&{N3fYR_;UY4tC$`*S+$^q6z%upm=!e`zHVOJbu5`Xc@x9(uZ?gM!S{2Aad z%Bg7h#cE~&i=d~VeZOt6`o8!^cJ%h`xBXrVol}~ZY&w?)Qji&s`(_)RDeO3T7*;!R=A)^97dy4a)oYW+nnXbgws`U!vzZ6-#xkx1` z2_~LFFgOqq8|~!8T!Pcpe79J_yOLNSLGR-FiYPM!*g@pcJ}pY=&``HA0YQ9qjTotN zP)J&WR*Ei)7K$!k+*BoCz2di#+SgAgJUz-yA%Y4-*tk@UEe(u6EH-{kPXak?3m$!4 z_MsjS<~1`kPAPtUcq>wRrR)}Z8>8*YeQMgZ^gGD}xQA?cbg*SB8E@E{`E?M4uM zYF;T50fFQ+!9n%3)kNN!N!Q!ew5b-t?1FweijLh*ME9e^gd%>It7Wzi=~`WIKZ3}@ z(J*TG*udzDwU8vI?%E#-VywR8^kg5(`*%lorS6I3FWDlmF01|eFbR>#rt96UNs*;j zjKI6bb!ClO$2!K+2A3U;RkUwgr4?52{h>VHXVJ44M}i6MmX|f7i}R~hITI}4)%U|M zAHVuO0;vaMU<6Nt+(uYlP4+C|08nX`;W4G}MUSEUP+F%6Ag#vvoX>k2&QdU+ zJ+`Y=uYt)n)Vc*zxrZqn&sOSy0Zkr6?Si1L(V22nZdd5aHOB*lfNfVoF*e09wJc;* z62YtZO~u*I*4s^Mr{raYF*&W(WRG5xbG?K_gOifH{`8tcDJjaQo0T|Qpzu{bhm#S7 zJ35&qDT$+beE}KRNaQe%&r0+XGcM^pAr8Mvqq)&`EPe!R!>YaEY8LBLc+@BuF=HlV zu-&qRw4Nr%qU>NQT$L4E9WgHsy*TD7O{93MrDpi{$Pksu$e}MpDpD;(gM57>{r=mp zpMCfGW%lUk?c=X{5@Z|Rh^`TWq#;4*o{?ZFr+)`P@9KU6%SF_VfM38SV2zrnYyGb6u3Bd#4AbJBpejZ|1t#GNfNS~m$7{jM!0bh?88U5|dpCvq zWJ6BPSX!}!Q&UMdO_dC$9?NVDrn9pbsix|vO|u{Eo)!EXrmh*iegkG~j{Dlx zzI3b6J`QaB_%LCK5ByI9Y$r0?47siTeB5K8&)sd5)?6BvG=}AAg81BooOf5D)PoEf z`Mc~?ABao~tUW3A7q~QRbfN@N8^=o=s?mos6Sp3~`z8q0fI&s``Udp)Yk4c;(r;y; zUl2WAE@K`SI#4GGfza>KTJJ)|<9%H;ps{uP%j7%{^&s-+!EO=V9usvj^B+BQUei^w zc~00*OD~6CLa?x9$ZPUw+Yn z|3_U{N7F(S+ikjUZ01|H1-I@V?P-zisFjQ?u=gflXtZttBlS_fG`9e) z{gB$d7=^DzP1NKrmI1ql9CU>Lm$ibX;-pR|z~09(a~j;`p63~-kAw?3Y6)M^((zH> z*1|B@Qs@yVCJ4KSW|=-usljj))Eb@2oaLsmE;FC4ekp%g9okDq*19nJBt^oBWeK2fkzV?>C+X}%Su@- z(0Rl5=Q6*$ym3np2Aw{Qq&}WRBY!nb`25TP zs>bTTiw4dl?;IOV$3^=wQ8W4QJ-K;75t)0u$64U^WPqxSy?D9m^*~R&+R_7->8?>L z!TUw0!Ux~eZ1s)cIvQ=rbtEbXHRSjx+B`4CM8&@XjuK%XD%wX)W$6kDYFI>v&JAYQ%&N+E=C9 zi*kb?eA}B8uOp2+QRyDMT05S>&t4p5kH7l<+i$WrU;pP*|8|kRt{m4Foc7l0c5Mrd z0J=06k`jMsD@PaIk<_HA8o|pNL*@$1OjrqQtk$SMZdY4F*~6gI1~SLP!%13hi$&}x zRCU5zw$tqJa3Arn!^{TyZ|#K;9NT-*caUpG`Tyc&Dx@i|3d6+=CfKN9IHElgbYG6Xs0w#9sXYyX{ljHWQw9hyn@OX$l9Bu_M zHwl~+Q`9YGmUf-6*L#ooP1Ih#Lj$JDYI|!trbpl`uVk=;?cA1h)rA3no9nfAw6vLh zkr=C_Cr#Z9iG6a+`_Q;0j9tWB-3AY9Q^S;ET z*=ri@J!h_YI=tfkT!z2ZehM~xT;K0GOuz*pVD^uKzjqqMu5Za98EZbv9q*Z5-4Vb^ zk!L#;hOnw>F81i~Z^dR+%+lx?#!?9yLtGXPCFvhrMT=dvjkftEMQSNA6H|ItD?hep zTmw?n#Hfp>D^UkL=vVU{`qhKXsiWM;lEwO4G?6kOkew`J9M|&Z(^Ph2z7~BpP~8N5jJj z>p2|Yh#|gEPYK6G|10n+(E9Kg@x36^GW5h@3f~J271T(H0-<$HcsP76^~rhlXr-a84XrY=Q<5Lv_MZ_wf+W`Ui zZcA$p8d_uWNsw81Dk*r^#s>o#^rr@4XtkpGzQ+=Xs+Nl-mK6uvX7tT!pzE-{2Xpkl z-u_Xi3+*j&mW4U8$_v60jo@@%?BKFwegnAR^IcV5Ifozwd0`Y2gj0(V8{$>1rcgzC zvM~nCeCyCD-j(7k|L_0z|Fe#A8Qpgw8VF%tha?Ud>lWgO+>Fn3dvA(E&d~7Bn2Y#P5ffVPGj*HMoiNS;IlYd-DM1 z`dDwqvC&IYcn`>|bSld{<*fTXD{u3@ylmZOJ7}9&!b=%pN4h}g0`gU}N2z00;^puz z9pGi4pjWK=@1V)+ud`O*KDie?E9cu{6P@0$T7ozu3whMI$4fx%=`4ipr~WXWBOZZC zc&xh$lhR!6o}|Dw^Z9jtBmN{fO}Gic#PaattlU&vh7K339~A(&BmQ2Uc~+r(E@b!F z4MPcFdUBP1inw1i-7R(+Q3E)r1hV8Ord42-jt6dgCvAslqrBWf_}xdTrY&Q8GH;4C zC_IQJGcWS1qO)S%#G>Zx6$#oocH8xCi;QIa{_KqI8)_zY4gjOE+u20M+B!ZtCP%i} zijtMwgQ&|n#PTO>CtZj;Uz&7a(NvqMJaVM;e^5TswOI{%Lc$#Rp~3Mt9xWdbqpOaG zHu#bMUC%-Kf?#4G2e<6Rp!G1&wsPV6h8eqe%#9)Fl6#%rx0r*vKaMi=4?HnVn(wV9 zCp5tCJ3&_1NvBC-wAMSDsP`N7R-40}j=VCpBu!T4Ti&nnmMzXrHE3R`FIx9Prr95z z=6$edZjV26dkmV8R_JNC_oA->v~^Lw6GE?6_$3UKX*pOI#at8-1Ew(Cv^Cu!D0Tdp zY7bFB2745k-RA&sjj}Ug#Wo-}5OT5NY+y|+`q+ezo3vWLcs|X`Jq+&-Q_0ds^A-TKgq5dv)&ZBiH0~)N3(ku|k%8 z72C{-LCr7TZRwEL+QlMOAgxea0lLZ6=NRL?BLaPKe5zkM>ITomT;9Q?4rjjSJv;ud zPM0I^%gpw4f{B@=^i5c}`}2-pKtLbd`Oy;;F4&p8El=u1z2P>Kt6#?un_n_uYsW1S zd+uiBUCu3U{M59n7?`eg} z@kyjgml2*jNv%4XOV9ED<~Ne_72duiUb4>tYErzze7hlDJOI=YP>jXcoXycIIy6eg zXxSrvEIu;vHi2(blQS%;#Ad%R-q;+O9vd-(M`CDg@GT|-oxIs%c6I&U5kP#Fm z*}0Ae z2KnpaN-&~XP|{W;{$P^+SMIvg49J)5{G$FYTioP}^&vp^da1zTZ84~@OP#FDr*inI zypmSh?X=L`GYvLWHu4XmI`1IqkCnf?(p;?M!n|)mRh^{JNgvweeERV()C+%EyvMvq3cr~i zxe6ucd4AkFLCQrvJXRgNZRUH-{VU&**?0NDJ&7X2`0Jg zo*oAo1>bV3bL0JbZBFu-n{l=llR9A>oS{ZbxHyG&OG>swhHelmXjoG=+0esUlE-0!i& z@QQ-EA|aX$75cOCy#&!mW9XzTT25IRwD8$bx}ZSiXJTzfSex>1+$1u)6~e1fR_q!0!N|7O!%W5mDqxlfBMOJjpd+`36NzTh-E)9JHfgY`G~M*KwY6R=lWsJ<^dX~3icC1R z*!Q3p8Zv8)_a3aa+P4`wH}WOi*yho)3kSyuBN_r%k&G{ku?opN6MoMiQ~LzLZ_0nF zE{=o4&(zs(ta(43U@N@_G&Z}};D*9s+loQhcL3h2^MQ_cZ&j#E9i8GDJLTRn?B6r^ zCQ;FM;%w`NhZ8Se8BelK(Jf8AHqJcH2{3qf=lMG7J<9fq<7q*Js!lwOj}+c#Q?dRC z;g*FM7*{yRzzzq$o+!~A$LR_jdy5h(*fxkUGiXGhYrYR18BJ^FtM4xdw0Zq?XtK|B z>lM=r>y~@m3wIS3f|u?i$1;7`KD6(};4~G6$pstPkbZ4#+{rMSpSMK`;@Qf4?yru7gR%zF?8BRLC3?+(1$^G-<_$Bo6n7Q z#KgAnf*?X-z#Y695z&#ITLBL_ZKDW^v9LWBzdf zZjr+eF|P}QfQ!DlbL>Cjk_Sjs7#rM1;|Tew)Pg|f8V-o#{U&~68+zk>wH0e;4T-b4 zG95sB;G*9o;~E)NTWbJo^P1N!q_1jCZ8|LE>Uul|ChtT?K{jWKR~9{Uv%xWB%`Q8kU7fRvw(E7b)ywvVPB*QfGvo3^wpES!3`surkv%hnp*qHE(&uiCm)qpkbR^3YE9;zymd;Bd=2Z3Fl6v=ic( zJ{1#w2L;f^ywTy3w*;^{61}?Uy+;=Fu1Avwet>}uZ)gF~O zxy>Ln2U|{hZI0@Cs@VeOMDtAB^R{hY5bE=e{oCJ8 z+*Yn_6luc}@ZRZ~0+jC38KRrUxxf2_S*620cDJkX;1~A9QEyZG_HqOXPxu}#A@A!8 z(!mL&O*d{ko2{k%p8g~J|W!T{qBhIyCVi;f!x;HecsQVg8yt+ z|A#&HaM$|3tXo9m9{%-(h<4)`D|7tDydT&`?&${+qH-6G5x4b-XtASp(fvcJZHJTd zw#W-@{sjARtGx*&ED37@lP)v3ucJhl>0`_{?$sHqDlWW>uYK=*-~Yn*j_%lRe?|O@ z-tzu^Ztib&x4ie|tewXNpj`}(J$nDG<_0%0cyM+2;Ci*WEH)3$*SiO}wK1_@ zy1u!KZybpKKK=M({8#+v{QL3Y!Jj@-&*=H#$A^cX{!4W5YYe~+R;Bp-zx>bq?_Tug z)sz1iz7T@PvMPpOBlP)MS!_np(Hb#79;OF9S?XCiFCGiAb5ovQY~L2|ws6A9pNI{# zDNi9qW2Hddu3o%&38C_TEvDNZ!m&+P^Lc?Wja8bTPE~%qm*gsxL~o>2L#5$xK>+ze zCz;fbsKoYWjgam1@@OfJylGx-dp-FC*-)=nXYV7h6jk8|@#rx=Ng}u;rreSl6weHx zm0nviN}c*ow?sYYUX?zc0XUsU%w?fd7md{mI`Jj#~kHrNXh85<|VuZ8e+*$MWbp7jiYBJJ9Rqpe6!lE zsg?O`mWctVF7l07nrcaHSD7A_b0{pORGHnV&~NGSsSvrHM=G@;AtGiq76T4IulHxY zHDb~FzZFaAaDmo^N(R!J0Hmhq_if(8@5;0_Jm&88bzrH18zn0#XVwZ|{(H8htv7}htlN0&zlM}Ej zm3T+I05r!3J<*3uY%4 z;f5qoS}TK^mlhfxbGKX;n?F{D*e_(5vVtwl%Ie@EAgAdnpYMw33+Z=d%{9Gab6WEQ z8duJA3^*Z7tVWn*;HVZlPHgbhk->a%3c>03xB{RxV{^;o#1@rNpr}}6KmClwiIYN% z2WTwfJc!}%V{tr85{9W#A8?vz21Kof)4=RSVtk9pfMP|@mieMcq;KzPwwo7#&trTA z!QCs<$FRgP6Gb+$QB5cS20tM{t*rh;pE56c&|auXVlkoE3C#w;OuzX-rdV4^a8e>F zQM#yf>H!IsOT&mKWP!_a9Z}OjInBkPyjuQo8!Zd5mGl?l;@n=`db12%*QRJRHis!~ zTsH&H6-MlWrQq;FYQX5g#6DgId&q_VACqTCBC*7yI2P0_j_FU5KnJ`oPEp5wGZ_;$9 zylH_Gm7nFP0qD%dX0?Qgsbwc|D$lJ^6g)U~2WzC{OyV6rBap^W!SPq~D={rZ$2@Vw zQ474Pb{rBs_csO#|nY|))zMwCOo1h z;Y5qpA}RWW6j`F{3!wwZ4{nM@E_NO^90VxL@ghpo*M_3y&{fgev}c}9VktNS?E5j& z+O+wpyTtJ5ve_pr0uvLFb_eoP^)FTy92Lqc&MAJHwbWyC@%p5LE@WQO)LED4;A9Pe zT-u(r5Zi~JVhD_^9AFqpuf|ML;T$GZQjTJcvDYeapv1Fv;%0Tmg}>d2WqpiLbohU^ z06Z~|S4(b4YWO;Nq~D8PU4W`{2*-Ifh2+@2JOz`FOX~89C0hU(mtOq7JIawE5k5VP#>)u7#*LK0GK7E3&zua@U1@EJQ!FIm}30*FcX(hzF# zxZ6=77P7NeQio#Cu-(1?1i(xC16ess+(&rYih(GWyG5}P{IBj-r`C>`0}9SP<};~L zqG!c`xXV1eeeI7R0;FG^RLP1g8L{zVU|1z@C57IJB$}ALFnt^MJUf{NB~cK7)*->! zifO7A`C7{G#7^k{@k|EBV@dS@|GG8Di*|J_>(^uv8&-0Hu7pkQ;EOo0Y)#(i4~jCM zjLk7JWkE)=>@n-jQE-fQ23EQ)P=+O9hI|q-q%fi~Qyq4N|I}`fH3V*M$u|HH3@B!Z z=anxn(IkISEU`gk2hQVnET{`VmiacEJdGSaDqoCMz<%IhvP?nh7uuaZDq zBHlcJEhUb*b-pp7qFq2QxJLjv!@Gf0Egjp+nm_kKuF&O$P~rGsU5fK$9L46^NXc(y zUSa)skVNeC8IABjhk+_EwsL7~Hg8(>t7B1qVp?}_d+}&|W=oUE?kL+%?`-IR9B6E1 z1wbmA*ag?v?vr^GC_<)6VKZAgZ&bNsXL#_O6XUAK6q`43&e4aXXkAV(#VV9JO9(9+ zZwJLncSQ`_WD*pF^gdnVeA~WiP>+bBOi^TDWv`` z{LO$nX;z4{Zvjd~ZH!X(q@KG_GBBQ2>wMatVkRI?NJ+$!Ak4E) zqx~UsxFJW}uXOCupIB!L&AnPk&WS?POJlDu*E2Qt{-!ie;;|_S^a7x%A&^wB8)NTD z!ukh#ZFEz~JJ{J&3K%E$6pluhgzKpa(i^*+!Z7Nhide_Um?crch%p9GR~BD`W2MiW zc;N*MF+fBwWvRtoxD?yCYLB+5CUA9PzTt*_vo`%^Xagtl(s^ZWFH7FD3`bTWj(EcC z55x53esmw5u7lExzCEO2chqg0u1MRJT zunq9Hxma;|X3ZeY(d*83#FAtN%Y!#$%A{o_RL8%9169nQ$E-#c2x|6B&9~(l*6^zR z0u^hQ7HiQ3cxn>pk}c&lCslE;TEuD7egnfUlln)ftg}X_u}ZSca1OGp*M7y*EfCpOu)R3L{+qux@#+o#Eogu`E3=Zw@R~zq zH+lT(drBn$GQ|1z0!>M$MO3`YA$brX;7bcH7+-7qN>zX#1ln0KRkO;DjqZLz2)n*0 z#D1RTYqSSn6&tXyk}X-{85c{4*qjkuF`8WEZOlc!nGLVYO(Bh1b;H5LBi1$~wHNwO z8oenB`x^U^TrJ8!E+dfvQ)4i*6%TCF?ms;@g1lR+t+TZ{4;Gh;1vg^*R;AnX+Dfc= z5OtJphxJ3XK^%G8d!}|voxBj!dztsEXsqgNg$^5PslqHHF% zA4Sm0nDlJ1xc(iqYyq>_ z1@Fl&pATy$B)JAzPhJSY<#E0pV(ODYv<49L>s8Gg^p)>gUJ2fl-m|h-nA$RzhHtc< zP4f*Vs0T7&Fg*tilC<#XexxrBXT>|oN6=r>xKuQgzdSUxICG}R#Yg_QII|yvHMFj{ zjN(2R>%~IJ&Y+upA$(g8vZas@7X5(*c_D)k2EJ#GR;z^w?Che%+m(sizX5kctVCpT zJ~x51s_gWJmIaN9_09q7YT2&tp0c>Y$fvHqJPK8+4YH0A-?U#gnE7FIn4fg5M=*JR zE7!5rEVq!Uv-_4@yKEXX?D9Q#FZf-AGv)rmR<`EU@}0khaHHJJN0AO(fJfxUY&>*R zmYuJ69HbkQ*^Da&PT=}fGrpq0aYA+Ed!vSdHz8^%k5Bt#N?KQV=R+x!->bh@sjhPxsCOOa zASf&tIp1_HvymNDFAf%e%K4i@rrel$$b6woD#B6Eg`V_EkAttwZ}vQRU{|=S`8IB( zS;m#7g=Jly!NEFlNx+AF_K)V(<|BYkLV3MqZhD8)$^mt1Ptx5Ij%8&B0M=h(GCZ9; zF?>!K+z3H_YTk5QPFP$_rCZxAHA{0rQeK)-bh6QW-%*$ZHRiU_HGHUR$L^*P_a?7p zUPLoG-!$I(jxpFK?q1R8uW^7MA50v0B&;H@*qoNZ(v8{i91Pk~1!0e}jL%1kT+WPk zi23Dq7+Z^NT&-vNxUg-?tg#4r+D%Zi-ncqlIM$sM48fp~@G!8vtt@CQXZKRj_Al?T zip`LA6`ee|*Dh{OH|313g;5b%l#r+C&0mflVB}xzpUS6Zq&+VQhSE~jr(N*1y=$m& z62)-v*)sKD5Tt=x2~~w{AD2nwql+^B7q)@JAJ|MNE}J~N6p9S;-f$M9KUJ zQ4N#C!AT-EcE9=3*hJ84Xjd=<-y2%Px1{~c_eg3QYClNGE=FF}mjz}jo)$CA{fRzS zU^f)XK)#!An--cS9R>n#t?5eFt95vA`xZT+_gcUA=xyL^3xo!R8{|fSGulehwKC$L zcTSsbL3;Dpo?KB^&Kieagh?6t(AfApTtP#V^T6cIi3@6_u(?HAs~cOt!j=3inPZdf zhey&l4#t+U-@;4BChrXw_T}%f|0rBtt8QccSGcgXea6g|Z5VS`ksEtY!j)ZZXzV@+ zSN-tARvwacJwODf<+VSSK&$xC32C7VC+f(^cxl6zv+w#i?0gQg!;~~uX=f3?wS%+U zvS?iZ_2M5NBzvuvG54@(1MRztnpQtNza zU)JtzOw?u`=oU@UyF~tByXVRIvqE!3BaF|H+qJnC>$wG5#!RDvLllLYH1Om9>ar=0 zbGg+nuyG$h4A(qY<*ZosyI%BS7wv|muvVpDkmA;#3bwvDEoK#Uio4(igP%RG?nQRI zIX;|3!)OD!0PaVJgXl|lxd!BLrrF$3E3ej#8LDee{jr=&m|?ea&w|=AYxVx+_y7JW z@n1~Hga?$SgG?v4g@b9qfBE=N2cJ0jFMm4t=y&{=e@gt9TOhhZ@5_+1 z3CII6G@I45sH)7!QI(zTD)7!i2QLdHK$ap1PXuK>gAav3iL74O{v8&EkA&^PE_{W@ zuQ}Rra2})#+TdHz3xM{u#^*Eu=QY;Vpg-NN1?{3AzB7s27esT;>h3 z9vTnmJ-kz2t-=fldF8H>MhGWSu+t@Cz{cNNbIO0-+0ca7#E{ASBBYwvn;fq8Y6zR9d;zdzw{+x zC%Hgd#7#!liK?+(Y{y{EkJ&+g!4~e;5c`$twp&5@*r3GzdLlnCuzHy~$R#0fgerP{=Tqt zsv1~UuQtUH0d@L#%=ZZ``rVMEFIw2~4-W@~owF#3ru&JY&Epd=H1{jrKrAdq#X zIgUbs@pdprm*~w-jE~S$@(2ZsNQf>5Eg3)z!2c@JWWB1kQZ69`poRFn*ixIDM52>m zIDNG()={;Y_RJ7@O6dj0CIuyj_C^97pS?p<_)sWZzY2ElSx(C(Jw z5J^GwhzgC>nA(I45zt*9jw~1tG&8h51=$cTXL)8&YMNw4!{`B%GQkq##GqK4=x>FX z#Xy45_t{BIMi8?tnzk!jNZr_cvEk_ZtRgjfdb^!lO{O22qMpFu33U@7w=?R07%>sG0RBZqtNv3n5D$MAT0 z`e;lz(WTv@m_3l1j;NxqHM;RdznMNqG6xmAD{uTP9QWwK1BSoX;Zt0WMpz*@DF)gI zo)$oenS6}go)-XYftenPO?~ey*0`9>84V2ys&w$nXflMUc2QM3pCw}T@6Q|d*rkp1 z1AO8ZOu*jlY6MJ6P6$fNDVXB4#l}2ljtK@~16)wxmPjTol^f7>1R13nkF7kr96gXl znL8;Gzw+2KBe}lrEJAJv(#9}}{j%E>a?y!tZ#9-Qgu{^6SLgstIKOx>4H2G}6=IgA z>BP{Af_y$W`mlgo2cO*{&9=9YJVL6NGe`|0$Gzy$YI_mAd4jU@lQ(e3aV0S_s~Yhk z`tra6o~ka&v+W32twW7O{k2j#q?8|@oKV}%&fw^uoD8CTy`Gm874A?wIich{r)7>A zl#L?=nCr<&ByXItJqST$wBI2Xzj-pEE9LXuYFEJx^tAX~^m&A8#jIQmH$ZAD)O$f6 zUkecv1Djp{(R+Bx{BS5*h@cB_U4R{gK@!cXibXCyG_BGW))`z;5+lc-XH(=H`$A|q z^gyEM1Ve>QV;#*9ti)<;+YKDN9Ye59n5JGT*p~5Jbaa+SODQFftP2X>|4Ee{&j|3Z zMV599Yqb-BySc{AdoKicK@;Xop=+`fkHEqTW|C6yvcDCxk?V9}M$jsPmLa6RiBpeY zI?8G0RCZRR$<0vT#haNS0(sGDt`sJ=Snri!hsDyRA+K~yN^wPjLwI1Wx0``h#!c5b z>B;C~*R4Q&ge=|2UNFnwOc@M7k9YP5j-DPm!h&=<9k(rTe+9Tv%RKAsxE|oYV=IK} zyqpn!+Bx|MIVaB~w4UTq35%#c1{J)S5>>3UX`wT*DLyq|e&0JHcb~I&?{n2Y9zC%k z)BBDUrh>wVEy^;Iq7>^`6py{dMhp6b9LF>Oo8>y@^pA0=vJgbfkU5RG)(D)n7m`pT zYL{_6=q|<(cvpvUQ|?-^F;skbT8GAQX4$5;DM!?B?M5%?cAIcV-{dwO#+de7YD)4* zYBPW^_fI|c(DOue4{*`jxh5y*vM;5$EKgE4KRlcf?1Bt>a8rA(%HkTvHCW$?unlZ! zwZ+Qiego?-J-!oTv$=Y|zngSl1Ey30omGCw{E?z#M)WCRD6fX202v?x2aw?jGBSKX zZg(Q=bJ8$Y7w!<`Z6+`As-qzA@BQvJPK&b@flu`uDIe4(3b;#q(;`GN5U;3Fn#ReA z7A6EM5lYE>O9Zaz&SL7kF61 z`ZVaF>~1E;p?#9uJLM)7N;mlTs~f&~E(Gff>9ubx_j}PbnvyZSPA>H7VYR&x>nsN} z_u-R=Un#|fqW`RIFB)4er)KgVhACvrHLPDqHnQH93nAu~(`Zp{HlqKcsgM=`J{6(M z-VV0g=RI53mrb9pDugutuY>fXKR~PAzWVO1-Z^4>8&gmT-4+j)fz1cSntAxirv|cn z30M~~xlg9kS$P%LcS=}ekklkPIBy*T$%!q)nEME9+-z84&>ZgPD0T)o34W|G+i}xP zLDJ>cFzIrY=eOQBl~Bg>H(&qfQwsF7-@ZJ0`RFJ!%GA6g{ALldVxn zpMC%0#fV)!fwMA0D2PnP>w7h|#^ctN`;f2g$Ch?Iu{(aZ*AN9?I|_h#2tDTG28JV8 z|Dealf-_&2{9p4Su&u@fN*E8iAQqVfL&bV-gfSOad@p(|6IWx7i0jox94k|};xnS^ z)wx6X%pjnN!vK8K0FzZ5`s;aqV_VqF%H!IHfjpSS!-gq#tF}Tj*@M?(!^#~m3*MK4 za6_c|w+~K*1D@LPh+TaXZpnmspGi@y9@eSq$qth%d8yPJb8km7t9j63a1ZUaWckOG zoqLCpqbPD+*xJNB6^jasWf9G&?NXMCm#gJa)*fh)o>KT)_vL&NbW7WA&UMZiCye!v zqvKh_aaY!E3F^aNy23u+x+eJ|h5LK-9jrCq7K#n)tI_Ad4`AfaTC1FwROb1UZz02_ z=)6#MKmbrcufGQ)fMx4qZnL_sr1-uxpE;zK%8O!nD5yY`CHUf@eh5-MS28+-wV3dmJs-pyfYXEn9Mgve6!?s3 zQ`BMPW9XEX9QmNFvh8F8(IPlR_az)`bDTdKKji{gOage0gu!HE|>V~t|j{ItB z*mYthUBmSfKL&0ClVFcwb++`lnYVezOSn7cQP|DbrsJg$H}zpinDC9Une}4-+cr~N zPkI526{)#80861rpk5f2rU8$JVa8+TS<-cu#|`f|&Cl&!5E+*l_i=CzTHumu5ju#C z#Gp`(J1gwBLZ#}Az@#ya+Lv%AoGEiHj>8Mn(iq1!{BrNsz7+2Xs>L8ZnWNH%`qYGV zGiJ75;$C|AvgOzi*6iXQVw2fYIw74V4(`a?Zo1k4KP(tnxE92lMlNUcG~8?ND&IkY z(Z-TGuA$``rIrMnuJWrK5-S0T( zi+o)iA56F_f)SVJ!iX_68jpZ515P|NJ|=k34LX$N<lYsOX zvJf-tkT1~Pvg?alJapN;=ou_O90G_L5=+ln6T0*0vHc76e7}y#qsm@ehXZZ|?t~*k z)N#2DaKy$*YwWlsagO6~I1wY!){#pCUJ7-2K=%>02MtDu-|MbdgV6Gzc_YEXt_mia zK;Cbn#4wanw~in{4bZ#`Gqgk_XkuRESLDNjHiMqS78p`KmVz#0Iv!7YauUH%BJ|-o$bFD#MtoDb)Y$F7Li1HV zi=Dni_kqYjWfQP#ePJ5@j2nJa@~A00T{9_*B9)Mb=-4;=28GN|CQwv#wOs5LE+1#- zE_C|*A`{m(D<0qf`%cpyqi(c%(7QYQF9T`iwrF2_;(tASc<}MTLkIusqmLhc@;m<5 zKNQO0}`F6FMOK`5~YCb0<%__}Lry8^=pU)|+fJ_(@zza(vNZdGk3wPl? z8z?LY{$&P~)n(?Z^KZcGqd0ouBY73!`+1~(n zo>tJYO>tgU+s%!IVdiG6d@2}XN0hNQ(%8TG4j*@g8KaqF92?{#bc%SR$6d!R(a%6O z6b~h8ZBkN^DuS;eN$a+F zw`S=}QU{$&9(FGIkSl49Cp~$5^!Tf%hQ}9I>)Y>NX3t-L_x-Ck+*_&}eh;Vy=J7_RSYYKa0M_RE~d@>(L))mQxAP$FpLFk zj)*m7CXp}Pz2WcXmeHw>M+w=r@&9XcFs-~}dL_$D#6slxm{HALWey3Wq|uY(x(TA_ z#F-)6c45p{sPXXMztvkY)C4M~7rxtBxQS;;dYx14?lIxDw33F_PSna>kF^`ZYBk15 zhP2sz1)jj31g$g39}Au4B~<%i!gu6XoyR@rhP6m6ip5lj%)3uN&V=w)$UOs|z_7<< zCQz}w>t7^dYoBJQg<3hCjo-ccUmcD{lxPBHQorB-s>o*#PFJ&=2iv@yN7P0#?}Gp( z8XLjes9Mj02@G#VLrdk zZz@dnMF>DJ%yQV@2t7y+yGjBN*a`kP!$1wM;mQhEATF>pdb5)4`SypepL~5JC~dYX zSvQK*10hI*YApokN{t%~CUE`=7mMr;+*UsYQ~FC?@QC^^dI48_FrBM2X`xpEDK*t$ zg>xnsztA3~Bjq%Q@Rq*vKuX2CVyf#SNI4 zEBw^z?93GeJ{$POq|cbH@`ce*zqIJEKYFfMvJ*c)8!+F3-Rli%m)icB=Fz>cyWm^> zm5`#lU=*g)Ll4D(1JE!F1n<58#AS9f!i|BuD~(=*LkX;M@Ge0fCxlPIOdQr7T^10? zg>hs(69z)4%ES3;wMN4g;X#5sC+uLc+%Q}!_P}^0O$v_ z?*(47A+tOatSf@~r-Uvu=&0zFVIQj5ZfGX$2a7q(ujLeggz5|)J#8k^OGKJZx` zS|!H{4@&TV#YKz(S*mtYpKJrRo>9jom$;)3Dwy*~)5MDCQw0>|hO>63sBC`0u30Q+ z_FgyW-M8u`dSr|h1>4LRFwJe$^2Na$G;AY5tsWoPb|y8o4R1$HES0M8RSkD+t;{<{ zs^oeg5Cg)st z<_lvSI-U@^QTH{O@Cl)Z!zfJitD6%#A#?%GKh1Wgfq~wAw*Ns2V~X|mJVG_6!Etk# zYOQ26a#cfevH7Z|`3rJZd%pMKtyq&4q5knWn3(!o@GifW&$UQF-4wWt)|+CBr)6Gc z<){Dv6lC;m=_CNz|3s`+^Ddx@InrSMemhDR&Z_-?rHQvurlk! zqS1ds3=)Z{sKJsh3^!(=_5wlysiXA=;06$e7hRp)udA1Gi1gdNEgX-(`}PNEqkQu9 z%k0_Luiw1o%Gnn@VRHyvhhj)jj2EnlM1TOsIrHip9VDhXgA;VVt3WtzQkYe6KaKMSq(+X=0q^>$H6 zYmhc-MYCdBRw3&kn#X>v zC0yzv_4*Bk4z11XR!97h*={PGQw5JigWPFxkzau|(l%W=RzgE8V9pe6Wfz|t1kM+q zgnRgesOW>pBViv=q5Ac4dBGM|FM1KhgKQD@hcN85{4K9iC9gz7gd-pTy}fu2x4 zf%i-K3YRpLpRP8La#xbU+JCIfi^(*43b&GUKoIqdyh1X7u~#f1(o|x7%k_RQT3_5$ z?*LYQKIGO;fEi7bQqD=L@BU0=Y?D$==oDAi3==kZ&8eu08BN{nn{dT zx(mMnZjEDwb*?x0I+f=`rSM|QSzz99EblA1b`Umv>uX^1R>J-QH#j}t3k(evrr4k@ z=bYF^7Duzb75`*2Xp1?^+anB#qU{Q|8ef&?c$|WH(;$R-t%nLBeMPgceF`BuU<*Lg zVv=uGUOKBjJzR-64BJ* zb~T*mfDCvojvpb`t`Rc35(0w|a8b;y;IePAx+;cMepW!-0TBZ-`tI)#w>3HWr`(4C{SI`H)e|#MGioC z2zg5g2|!g)(x8d3907k?v{UdyWK(unAFulAq90f;mclfC^hw7F$*ftM84ImVw zy$eZMgK#TSfb~PPEF0Y6r$MtDZ+?RcidbXSV$KiS#|^_KM1k(Zre!R4*pO7$VGEPO zo+2%4*dPtpUc-iYyCkueVdLUd6??(XxJ=0zmX zCb9v4t1DCB0k^JszD9o`lB<;a+OT$$P-biJ-a;M03idXK=wHW<#-rsMN`m8E=<2J} zu{#~+dG6kC8H3&PvZY89i9N1eDFmDixh5yQkZ2G{<6?uXT9czpjUg2nYQ$*Hon)XdK?q8 zGLFNooOYVMhbD5FZPB)r(~={+$!zbl$6iSOL1Z#O@A%<98a##WMc<1LFEPMN+D7Rz z92sJ49Y(bXmqg!w_qJ#1(UxJ3?pma=Wu2cFQkBNdZ&>Z>ag@NE_+>;J-0D2dQW-}S z$XxlU&xou9=}mhsAMbdSHf$!JVf&1lVbK0Y>AXSejbX1n*2%poL{5jtHD7JD1n zlwr(a6@jzKP)0d1LH7zpP~9TCa1r_i2f=8+brI{Wmo{c$eP#Yiw2qAaW~>tKA8qX! z{r~M<>uwuImfqidijIUq*y|>B$zU*`Lu2jOlL;`14Lg}EAP5vCwxr%rB!^8})_C=I zA7CGGpJeM?s_N9OyLE9gvuk2^Vv*I=bvbqFT)#s$XT%twoQq?=k;(<;oYp-HJff?7 z5#YW$#@gjzONJ4`0AYHehFi_R zPosF~c0L7@yY19rFn|+UV{O`DIx+)SQ0$>D#qKbH#_$d70VQd$=5lddoxz3=r2{@hBLN;>ekLsN{e6;K_#Vs~ z150TCR<$SuudXfWgR2&jRzGOR<8d|Vu>L?6w+6|+nXoQ9K-7wae}>mksfVoEXo(LIr=fEqrC@M!25zYcb-TZP_08+=-~9CBtCy1>Uj01L ze>hW0$PO+pn7H9BCK%wK%cnWa5soMn{?#>kJ?yZYCXyao(~SIJIGo3`?4WStd~D=e zR3C|UMq0^qzGQD0=rnZgpk^~fa1}_AyTV*9D(9m&`Ky;yhDX4aZ zO&BWVl0+0s`KVsxBv-b^a-ZqDLyX-d-Vj}w@d+bmnz)d|FhE!F}9Uzwf#va$rxqxen}Gn>H!g{S$Y22WolA+{REq(3j|^AN0MqCj`9Q z#I&JZh$y-&XElatLs{Bbx6jh!Go*>=tfr??!oI0bA?s5tIo51IggH*6HzYosL3%Vn z%-V-kC(g3iW!=qwB3((Z_<(HJq`Yy63r2c@5a3ymQ1>vUd3GgdhLSGV>k1Pgv*&Zw z?qUmtcpyqf2Jm9w80O7FR2y~Ek6l^M0PbeC#C4x-l{9xYr2;@1dOnA}i1!I!;Vlnh zxVq!S7@&BHVI1GlRT=H+Da@_9n?a#SHQ2#&Wn6GKcN%1bJjM|eN-qINUImeBit>MW zxMU`6%JYfj`o!{m0!}XstIotk!8(6-MI$Ou{8v9Os5v*NH9x31N2n!Fs3ljZHD9PT z-={t2r<09Z{t&95_SZ+VQ@7Xm?5C{~3Da}Zxa`;NaGpwL;-q2Bg~rbN-;ATQXcfEAp}zeSA*nnxW#ApR69Ettx`c9?$gI7k$$ zH&}}I%bIleI@jLa!(sMr!y)FeRUrdmmrO4qSr2TjV7Yw!iucuWTN@yCnU|LPT~Knk z#>JPi3xKgvkKTTu^N;)aO5Kpb@^He0y2ITuP;G47PIvj#1ES^=UZH;dVlY3T#Q|2E zYb{<;)$p0l+E^^F!QJT;cA#8fu2~Et;*nGkdaE|rfE?*67v$P>)H)}!I1b#02p&=3 zKAS(xk*B&h1KU>{sOJz-E!4llwBJHw$?*jq1B{NUD-59(_B1$nhys=DR!bO`?mRee zpZr`XGr{WoR|%|YFKWzrn)zbzH2oA}INVQ-NzR`qBpAKuG{XMdcH4Gl^5EOM0zvGE=y1?>7w_WB(IOAI zh|o&dPt_HB9=+?O{Sy;zwrh6Cma9?H1#fC$=vx)sTk?hqq#AreUc>6n|2R8W zt9zMynfj)K$Q8!KEqRX85ZB;G#BE|-NU6QTV@^#kwV?;h~QAe;Vk32c-Q! zay14k@;;(VtIcPGexQVdr7p`BDrs|N_#TX>U|B%heYJ)tB<(x7yJzE#7E8&Tx`StZ zhYj5vdFIGtRjCmUUYPtzQ?Yn8tr=~Ff)$u0DbX0k(KqAq1Pw?#P^c%C48^UK)7S zcO2;h;*YDsYmPf31$42D>7Kdf?!<7`B^S}cY+(lNa7?8z@vi~Zc5N4Jtd}Hi5p}HU z29wN@Y5di?mnBqhscQ%|ct-&MQVay?v(fXP$mof>1Z_J97_<5S`wlx-;D*z`aJh=4 zy^v*xm5v;DS*BqUK&vj3fZhR{t(sZx;I`6Zkga^E5Ez{Ny{adWg)!k2AkJTae!n*3 zz!Z<#WVAz%&-9PIUPB7GZ^-dL;Pz>;oOYvL3*4La>EXStbf<}3FTGGMW?NU>MC{Bi zYv0cK4xRN95Oi{+(G)#O%afboUN%Zg)p&6z975JTgk4Y?4twpUkhG)3EFE==srhvI zdz~SaceNQ3*wRv+(hXJeq9gKzEzlq9QL-X&xlD_|5F2!-TMi zu}*11SU8!XaBSTt_|_(~n)1h3P|J3a`*K3|k;Vn(kVe!sq08tmfjQBu?05BTFJ5e! z-j=8t6?yk>3+O546|i$_BFO&I3DUh?sC;bwL5+Lq{n&8tZX;&mGq`?WP5~x z+h6peiMDu?%*%y%hvMm9jPbi*X7VnmYK%Zr^IEQz(AigOP9st6=}p;OCKB?F^68ys9@x9)CDC4B}-~W3lB-4$uTo&eX5d zH3F~en_y))3ZN!I8jV38H3fu@%&ffk_BB5#0$i}N|u>02AqXIH%~r%;8E5mm@O5 zU{x?+6~)ciqA2Yfi}i-Bn%5?1|EZUT2O2Dk9Y_{ip>Uij21-XO;GNyM<_2n!9wF~k zuR9K95ykZ0(!0JFScM2Hh+t=6TT~0M*bo6os~>Ea4Ze08e#yyQ8H%EmA_UBcUqB^i zO!U1@0$?-k6r5Giz0T+(AB#AyBS1bKoy2dG6W5++*m<|R$EjXNg2$r#u<1kTi#`{} z+CPr+w%pmfBVvO39*FTwwIB9HTHsu$>j|BIjtvKmIxuUNJ>?We@yk(^jKT*;x_s?X zvXE)P1QRI}n+=CUPEY^OmR2kPCu_<|k09upc>1lK1nA2C=&qZbcE=kBJHCxb)^=q~ zVP7Gi;SZD38ybWf;4jre=?^&bKK_{F0O0F_T4N5Y>2#uFDwd_LDpY!@{HbIBPWc1o8A3@?I;! z(sD^`{)nt*2&4P(f#$&Oy)Low{#}~V5gN_I{EI> z!FR{0Q*g_?oD&xt89AQD5ta|fznd@hZmbnAb$m5SEte3izq}|HC3sVD?$6(RS7g)K zlxnUq58>QtO4Wj4S)It?yLY=ve*-=b!?T=5{WaZe6q|u571@p*yuTwmRKU@257`g- zcJTL+i>{&1muT&G6Zs-FLT(z*!;mI|vgm)926 zky~|L_(Nhpp}HCQUML7k=+vSi$o?`6OC6k%RRy{Vl%71PiG7r2N5YeWrdPEYjQRX; zxFlEQ=5jgX^q=$WY+ZtN#O~tUDeC%RVm#&0nf8tsT-YO2@^;AE6Ed zR_7ztrf@cfpFqyN;}gM-Z|%GnlZ{DO520i`Lx-He0Mw+`$PSOaP6B00Hf~X=kb?pu zH+WrDBM8|aXZ7~VWr=r{aFSyWT6MgBTqd-jXnI4nfKTO4qURgb6~*@hST|q+V?k^E zMUl;5I%BG##I^W}0Ke^C4lMtub<;^L$_3)0PFfv^TnbubF z3Wi;wcs6*#z8e<*A0S;u0nJ$<_ zGW?A;Ak7H9VO!Og6u)evHAdO9{ScH0K=joE*cpckqS{M0(g0>O<8kCg=0X^mPN2Z5 zi7>>b*1jZStzBd*qj~gEBN=5!G7j!o$2I+US;C^OAsTcATdl@}L5Unb3HyfRW~qoQ zMoxTXG$MG!1=fBfY&wBTbYBRobdN5iRW#B#`Y-@4Iha-ux@X`dgMgjn0Pc=yl^l-1 zxavmQiD{B}bG1xk?6Rx9q$`xUw1Jz;L!}i=ZmuinJFVZbt1)l;2s* z5b=XNLH64gf!D(8`M&>07Il@_2)bdeHgU$$pZw#N| zN)x*lrmCC(@0s|#$3#gBu)&r4ejS*OqcE)R`V!&!G9MGn~^nw$E8gWj~8 zG9w(lVpooGNt>XvQkILeaPU9Jki)Z#hlV)FzxLkXor9o)(M+pc2a2z5nvpur;o z$Jg=D!FdEYVRbErAH*mMD)D<+L1R*L?MmG(LA|h*L7lY|*bGL)=GvrH*O#y1?1F3*Tq z;I_wfEcJYf;mP27`vq;}T4n*G(J3%;BaG;HO3NZ&gfEz>Se?w(Hc2Dq(C2j*0 zO4MhK{<^RJe%~vB*FP`<#FPzT4S?dKxP+Nz#FP-Pi^f?_U^dVSk?_Ck+?a14!MgAd zUa%w&6Ffe8c{iXt}W9`SP1Zqh~v8G zhYX=J>K#XH68?so<~0}$%`CxlNUL1V>?RB%o+C6X((-FZQD2*Ja}A&txQv|MT*uK1 z$4>x>!-{vKUy}p-`&>#!R&WYBmENRRF}G3HFNYVyB0EGi?|i8Bgoa2a?;~~CXJD7T& zXbIpMK*{G-IiFD$>y{wz$_>Kg<-=fVj*AORq+9TElUn0UH|pe)XNgKfXYXsTQKpqt zC}MH8q2s}E``TCO#97+YgsF-)q86Civ_fpa9eCFcdPFy9{F(uPiWqR>n1f?`2E#+U z{{Rq7*dPsrR@P|y&-w`7%OiC-yJfWV=|v8nwlj*Il&1$qP@Op?J|GVot9zLap06)z z*OU_Qc*i#;XYVEJ0VjgP9SVJT>@h%(!9MDtAF|?J7cVPZYXGp&HD7EeST$~{(*d&O z>3^s#j7U?~yhux6lEh$fU?s{Km$33X!_1AKKQ7>Tq8I1{o?BO^Tef-fs=#aDR>5Q> zm*uBqF?sWSKny z<0iXs@L|GbLw3P+$(d40#&{1^1b1>__C}V6jBUMLFjIqT>Uz2UMb;2SLFQt$oq!}U z0g30jocXx{IEjfq)Dz9=!UgjfSjZSWc2mTcJY27^p*%T$=}jYRH;wrpO}u^tF_}=8 z_9NNFAjr-M=TgwKn`#_TyGI3nf{94EcikZE$tGw$Fr0tvuc}O|PtsU)X`mQ@?$_i> z$7>OrnD6LD9oad^sL)=}7YFE@Q2|T~rkL6o*?5ikZYeP2@ObrMMS|0GqdEsTypP^H zZeyBGA`OQ6gza=wYB+<-oz88%k34p?HSU%f7ZAqZt5tkz<{d{Pf5ncaCxikB&o+qv z5gKmdtGk=M5FswnCP?qTN^_*n2vCvI`Vrzr>46)ps5({G)*pVK-X!d3Xs^`7s-*@? z8RVqe^iNxqpghrwNTlUB6rEIz(de1E(bWVU<;8~PM97{}<>xzgFE3G)|hsx7s8pv4~8(exuIzG(1_9N^ux}Srt&pMvJ z$S4l-yY@sJqMe_(SA?+)-G!;N?{+f}WmxK$W>+Dcw<3c|b+KJ;A8RiU9PRZdKzV-Z2H)ghL@LLLthm6+y{trFe#XTaR{N5bp#PG7A$ZLeF>|dtnD7JNnbQ{ zMD^$|lr&JYRjf)t@71xD4P7}k=_{uqJ?e!}kzQA)=i9TPw|;opDs^)^ zpD9U(Z0gt#qU3Cr%j~qe$kSBUOmX|Y`%zf)Atlr6{;`%TBGM1V)TEhC-i0*T(Ena- z+j0$CkE=~NR%jlp_XxId*wmY$zc1_TvYgKLHq&aJT}~G>s4jL@)##r^k>h;XMCMFE zixL31Az633VAn3jiSO}Vo-J!koM)_a+<+c z*2DQ-_D2v9_M5rre0gzEu7^rp#j12*Sm_`mkO}p7^(dIKn#b>UKWKwiyQHx5#}@S*MLs6n%)??fzVJziMZ=KMQeA$Fwol8EO~K%$tFS& zzs~-hN=;4e-R#Bxue}S_y=@cQR2?T6O!H~aY8BTDW*@RAV_nx#Xg*wG5Z~bjQuRGF z7VD?@s2*}{C=ej9o`IWB%l99H9vz$(8FY-knN^3>8+6I8oa_x)+M(#|MA`$7s^b$a zhzs|?;Y-Tciuef}=D!=b1E9mFo-ZJ<{c|=B8~sIZ<>I13B;I*-UP7IFEUIvAyamww zimcCodp`8H%A)kq^m4OtGK7rbYtxmIOo$1 zI*vsjx3P(E7dVH5aMU*5Tn>_;%CMk=wa;iQCF(FfUYL12(Fw)MP2Wb%>w)Di@MKl+ z%&ZOAS*M5-zz;c7)YD)rQwwS7N*s`gABfAKSkpZV%adr3S;~rEmwy9I)k~RjGeNvW z5(2*yt(~Y7EwU?#dVM2-@4l>c_?rb3Ccg3AOUy2XXnM{MlU^{5z8tPV*o5W+&Ca&7 zssH4B8GV{i}1ndn~UZb#7Zd0*I0hn^@aC;;^59t!~zS$0MKuqfAi;8q6ETNtU{<$ z|M1iIlW*U?{^`xTv47n-?WjL)$wUH-b+Q7e`N{Hpa@hAe$ip{L{e^dbdHv(Nu}8vUDrqGt<+v}~aGAH8SQv5oR*cXed zzF2KjT-}eYZD<+|oNa0EN5!@@cS2qhZCgSd-)I(TGFt(axuwS69_yYNE89@=gf9dJR?VxcrvtfWjXegPiQ1|6t&8 zr)TKhVi~w@4YJ2&6=mK;O;}UQy>5)Z_IuEkBv3MA5^oQ=4tmiI(gR6Sl<3Bk&TQ-@ zBs9?l{uqDW3(9dWY4GuYAkZ~ZN2mhdnWj3M9!#EG-jNf_`ydZN`4gq=F6Iy|TCdN> z<5|7&1lvSr#gXMW6eQB`LMq~^n7XD#0DJ96G{=csDMvis9OfIl{}aP3+6cDjM36-n zaw}S?skpny(pl(;_}$Y@r0+J_%KA%_O@uW#*+y8JlWiP(MY5ISZE9^r(~C4qoC8n7 zfFOIs=MW{yp#Yw~0d&1P&@pvUJSaq@R7DIE>>yo)hC(rZi?AToIO(W# zpLGx6coHjL<%NI=d4y6v?kJ0N5-W60N<$NoXwuNi*}(|xN0azq%5i$-GZUrV5#*y% zij*yn>1s>5tdd`(GwT!LhADJM71b4;ypsVv%__4NIS@0!-@JYG z&Fk;q{Pg3imy;h}{XEfsPk(ze-(8l7WyL!q3wS~2NQGZH()0V=ub67#2~3(Dq@8gS!zHm?~sY7(9LU|l;w z-Vau#A6JvsgD$#W`bK?nH0qH~qEtez+JO|cy2DBHz=l_%=6#>neHJ`^b-|;P>E@2o z$v=i_?MpG>jdfe3Xfo3D4$@5rtZkTY~-&FKx(fqcrZh18DpIuv~*hf=R(g!ahzSO9}v)0ovm~MY)F*dYmP!u}@b+ zp9*3GPy{)o?(hq_H%)*$!D^4g7bht?IaGhR<90wIZ$9JJ99cKmfk3uG1=1zFC!j-4FvW zhX_}kJ2ol+>W|6``jM)WE>$vFtQ0#kyEimt?H4Pjs*&H$RJiAMC_e_BVS#&Y!9XkW zcls)Tc?UX2CFY|GT03H>4F~(x_9nZKrOs=$B1lfW@75pS*ISL`d3UO@2?2OV$EHf4 zOK!FrmnA?iCW;_>4dD#Uw@2=S$Fb|k6g`)VmwfOMGsy3gJ^sMVaVKls?tI8FQ@P_0 zP8;u9HSuxN#UGL?o`+D3=0pskn)+ha?;DCMI_HGzXi{)jRBH?$)gZp25fa>_fd7-& zAq#IHrivePi6^$(XB60L`Bm;02z{$m$?7UzA{3VxU2+B-sQhq7Vn2A^R`cls#Qd}A zYI;^}6p!%Bg@TTdeGJBRtII_}r+LL3=3J5xS zGhJ`cbP_GFcfK4+_L?oF*oA}b zH;7!kXN%Nr3)a`Ta6rjH;MJtun7*svxiSL7H-h=4zsQS7QzIs{Hzi>q*3Jl^ijgkV zedfjG&=9rNa6O^Wr&0D!e?#S6idbO^DSDAS=!st7)L*V(N9UzZ+%Qj{XdXZicGUul zUF|6a46s+#zt*Kr*jQK_<5t@PM#|m_}u>VHQS95kV5AKR_>>8TmNa&7t2WKKR&^VIyFQ z<;GMJbyqFYgAMV*)yKB=(0kBuc={WwTW!kqX1Sazs=l07i@raHBDVTow}Sh6#5Y_m zSAA-wE8+IoWHy z|7I5L#TN2a*!wolH?O@GbIKvKy9mSx>3od8xR-8J?Ir zTTIvBUxVd8S9<{npQoipJ)N@3g*Moe6XD@LImr}-VrWslDW{k-51}-eM}mhR0+ZOY z1z|Z`X_w@y!7N+|k;XPBV=mwX74yU}0@Tr5k&TU8_dgwVyLUC^r(E8~* zWqg}@S17Xaaf%bW52<3A6rgl%QR~}thi1Pg+DHhS{$VWqLz5>Z z`u(2tKyr~Gahitu&hKeXOxqeo+U%0s=QLJ5V0S}cMokeGqE%O%^n{`P9^jMlqaL<9 zisz$?#S||0qYF^%z-L1?`UzQEOpHVv<^au%B+1z&i5(C5IWAJ#EA_#b#84@cHjG4( z!r;JbXs?N&?Y=c9RE3Z?Z4PmasN^NihDI_vBC|fCX2&wD^Svc%52O*qPqmyZemZEY z-g8lq)!pm|r5@~MLOp^(JdLGFP;?Ms?Fd4$TDY=?oz^aBsSCuOvno z>Uc3x>*t_)LJ+_iATC@l*GhQ3VxuvD5sZL$D-r;9k5)8?Ise|l)7}kUYc}bS%V~1p zw4%AA=SB8Hi5+Fz{vU(B|3i~*sWUYgOv$hV8vC9Pv46!|6bR+Q`p+)YG z%$xL+BK=yoIvDBx4-0k4{pq{E{Kfxo_|W0Vi8yfKA$6m#nj1Y=b63oS)$aqiOyg0S zcF%wooFgSjSsfqs|1GrUu?c-NKyYxnmOP{3niHJUa2~}Et4N0pT<8m6u#|%t{NVQD zxi~_P8sa)g{u+x%I$Lek5OY{D-{sIUSE=d*FMk$8-3sm%TpM?uiHF=f2A>OF+G|IC zYjg}po!cN6b#m3Gyy@M9govygZ{)64!Yl94NxV(4dhuq4{TC_0qghDu3KDiA#?087 zn~BgrY>q)Tke6KI>(W-xT$kn`b2a3TNr8GC4+3$*PRIWAn(|M^?(AAJQpUmjnxYcD za1xrfd%PNL{>EYu1nf@d^G7!uJgUmy$;hB@875*8U*W~6S#|sQd_KScKv~JUkFkA_ zcBK~md^;xtn%8^wEZgUP2y)7SgEHw0Twj4p>VROx+1(=NTq96Y)$)4D#_$gt((}Q9 zK3$w$g6NJ&egjTKSXUdJL!i$u#b1EjP*Vzon*lKe%Rbl;ozw~w=^j}1{5gM}a{o*U zSK+s@ZcR~c9o1^J1@j4|cp#wd*r~|QV{^!N)>R6RS>QuGNT^aHM_yOUy1o(bnh6c3 z$o6wz^XB$re@eb0{4=e98^Ip(Q@>G-|qqD9~{eS^1f0!?zm|PIx^~Rio76T zq0{pV>7q}o>V=Ynoe`R3*3t3sTq!zz{d?26<;C4}5rNSW+F*zON?StaRyWKxxqo-a zl1oTFZ0d)?d>y)})eHtTAYziCSzdysjyC7kJ<%cSPuTMZ*CzX@{xbc0 zrEbhqb=TFx)y4U7y1G5vtNrRo8cT|8 zvN0Y)90B%bz?eBa0&`J&Dc)R`{B#QAqVeLE+Zcy0Yk<=cAdU962Hyc81K#{Cx~eIA zpI6eoRKY_wMuHy3!)$jkaJ^h}KHqA&s4ETPctMapQ0gtRGf&Cbt1G&J_Pavk0}VA!w#2s1LTxm>t>*;0`w%B45;3wh>@0E6qQ|+PO1bR$y&=e zHpnIE1upI)?y@AuG0;j{NK3Gft67Iq!#kKdhMCBll%_B4;ZUZK3G*yE{5+li0*P;- zpbBYlN@p!l&oyBv79U4o^6G%Hl5!f%A>Px6aWBMGBOo1evTxASo2?+Nb*5KUkuC_k zSfmxJsMK2w+S{(J1`SXB`G5ZFzh5#WANdrd7-EOU!Uy=T<~W~kjTlj5;bC5b4<{$X zjHO()+3z8QO8uxv>Ey)7G+g@>ie?%NfyhcjMPPJlYruo=7VoF)YPwKVCi5go`i4|} zKs83Miel?5WSQ6`JiVC4&u16o`v_y#?qCHwcR2-t&V&t`luH{1dE&}N|3^d%I}m9$ zX^aoVhHMHDMhRzHcQR+#m7~;}#KowSa7-7bb@C$QN6gAOPsA=hoe+fU=ClqjI@Q!F zwiucUfP+?Wab3|E%Wf$y#60QfeAW=l()VnTUjis_uSWpxuWfWJrd!YpzF6Nr^^yX1 z{L@Bf4UQQDcD39narAS$)uhIU?i;zm)W+H0z`P`Omzw*AsMu~>12k_nJ&O4au_NJ_ z7mki|Ay39=HEs28q}TSxVno~iR6xA(NB-sW=kw?D=kw?D=kw?D=kw?D=kw?D=kw?D T=kw?P{O5lG>6m8P00;*FDv1OJ literal 0 HcmV?d00001 diff --git a/runtime/glm53-spark-mtp3-mesh/compute/vllm-e02-to-compute.patch b/runtime/glm53-spark-mtp3-mesh/compute/vllm-e02-to-compute.patch new file mode 100644 index 00000000..c3301103 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/compute/vllm-e02-to-compute.patch @@ -0,0 +1,4587 @@ +diff --git a/vllm/envs.py b/vllm/envs.py +index 4bde607287b72e2617c8fed730ab31be4d443738..fbf2ad74dedf420b2e94aefd8ef03955aeabd670 100755 +--- a/vllm/envs.py ++++ b/vllm/envs.py +@@ -190,6 +190,10 @@ if TYPE_CHECKING: + VLLM_HUMMING_USE_F16_ACCUM: bool = False + VLLM_HUMMING_MOE_GEMM_TYPE: Literal["indexed", "grouped", "auto"] | None = None + VLLM_B12X_MOE_FP4_FORCE_A16: bool = False ++ VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH: bool = False ++ VLLM_MTP_NVFP4_LM_HEAD: bool = False ++ VLLM_LM_HEAD_A16: bool = True ++ VLLM_MXFP8_LM_HEAD: bool = False + VLLM_B12X_MLA_CKV_GATHER: bool = False + VLLM_B12X_MLA_CKV_GATHER_MIN_TOKENS: int = 16 + VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS: int = 524288 +@@ -1641,6 +1645,36 @@ environment_variables: dict[str, Callable[[], Any]] = { + # Gather DCP-sharded C4 records before B12X sparse-MLA prefill. This avoids + # query replication plus the per-rank LSE combine and is opt-in while the + # path is being qualified on GLM5Next. ++ "VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH": lambda: bool( ++ int(os.getenv("VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH", "0")) ++ ), ++ # Quantize only a GLM5Next native-MTP proposal head at load time. ++ "VLLM_MTP_NVFP4_LM_HEAD": lambda: bool( ++ int(os.getenv("VLLM_MTP_NVFP4_LM_HEAD", "0")) ++ ), ++ # Preserve BF16 activations for a runtime-quantized proposal head. ++ "VLLM_LM_HEAD_A16": lambda: bool( ++ int(os.getenv("VLLM_LM_HEAD_A16", "1")) ++ ), ++ # Reserved for a separate target-head experiment; this arm keeps it off. ++ "VLLM_MXFP8_LM_HEAD": lambda: bool( ++ int(os.getenv("VLLM_MXFP8_LM_HEAD", "0")) ++ ), ++ "VLLM_B12X_DENSE_ACTIVATION_MODE": env_with_choices( ++ "VLLM_B12X_DENSE_ACTIVATION_MODE", ++ "auto", ++ ["auto", "a16", "quantized"], ++ ), ++ "VLLM_B12X_NVFP4_ACTIVATION_MODE": env_with_choices( ++ "VLLM_B12X_NVFP4_ACTIVATION_MODE", ++ None, ++ ["auto", "a16", "quantized"], ++ ), ++ "VLLM_B12X_MXFP8_ACTIVATION_MODE": env_with_choices( ++ "VLLM_B12X_MXFP8_ACTIVATION_MODE", ++ None, ++ ["auto", "a16", "quantized"], ++ ), + "VLLM_B12X_MLA_CKV_GATHER": lambda: ( + os.getenv("VLLM_B12X_MLA_CKV_GATHER", "0").lower() in ("1", "true", "yes", "on") + ), +diff --git a/vllm/model_executor/kernels/linear/mxfp8/b12x.py b/vllm/model_executor/kernels/linear/mxfp8/b12x.py +index 48343f3d694b8fe2e5ea85e3e173c05f23b01c0d..7ed0b0344008374c9e00199730601230314f1f4c 100644 +--- a/vllm/model_executor/kernels/linear/mxfp8/b12x.py ++++ b/vllm/model_executor/kernels/linear/mxfp8/b12x.py +@@ -12,11 +12,14 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + ) + from vllm.model_executor.utils import replace_parameter + from vllm.platforms import current_platform +-from vllm.utils.b12x import B12xWarmupUnit, reuse_packed_weight_storage + from vllm.utils.b12x import ( +- get_b12x_mxfp8_linear as _import_b12x_mxfp8, ++ B12xWarmupUnit, ++ get_b12x_dense_activation_mode, ++ reuse_packed_weight_storage, ++) ++from vllm.utils.b12x import ( ++ get_b12x_blockscaled as _import_b12x_blockscaled, + ) +-from vllm.utils.torch_utils import current_stream + + from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig + +@@ -31,13 +34,23 @@ def _apply_b12x_mxfp8_packed_linear( + input_2d = x.reshape(-1, x.shape[-1]).contiguous() + output_shape = [*x.shape[:-1], int(packed_weight.out_features)] + +- mxfp8 = _import_b12x_mxfp8() ++ mode = layer.b12x_activation_mode ++ options = {} ++ if x.dtype == torch.bfloat16 and layer.b12x_bf16_input_supported: ++ options["mode"] = mode ++ else: ++ if mode == "a16": ++ raise ValueError( ++ "b12x MXFP8 A16 requires BF16 on SM120/SM121 with K%128=N%8=0" ++ ) ++ mxfp8 = _import_b12x_blockscaled() + assert mxfp8 is not None + output = mxfp8.mm( + input_2d, + packed_weight, + bias=bias, + expected_m=max(1, int(input_2d.shape[0])), ++ **options, + ) + return output.view(*output_shape) + +@@ -55,11 +68,16 @@ class B12xMxfp8LinearKernel(Mxfp8LinearKernel): + return False, "b12x MXFP8 kernels are only available on CUDA" + if not current_platform.is_device_capability_family(120): + return False, "b12x MXFP8 kernels require a Blackwell 12x device" +- mxfp8 = _import_b12x_mxfp8() ++ mxfp8 = _import_b12x_blockscaled() + if mxfp8 is None: + return False, "Install the B12X backend with `pip install vllm[b12x]`" + if not mxfp8.is_supported(): +- return False, "b12x.gemm.mxfp8_linear is not supported" ++ return False, "b12x.gemm.blockscaled is not supported" ++ if not hasattr(mxfp8, "w8a16"): ++ return ( ++ False, ++ "b12x MXFP8 requires a source build with dense precision selection", ++ ) + return True, None + + @classmethod +@@ -89,7 +107,7 @@ class B12xMxfp8LinearKernel(Mxfp8LinearKernel): + f"b12x MXFP8 weight_scale must be 2D, got {weight_scale.ndim}D" + ) + +- mxfp8 = _import_b12x_mxfp8() ++ mxfp8 = _import_b12x_blockscaled() + assert mxfp8 is not None + scale_k = in_features // MXFP8_BLOCK_SIZE + packed_weight = mxfp8.pack_weight( +@@ -100,6 +118,12 @@ class B12xMxfp8LinearKernel(Mxfp8LinearKernel): + getattr(layer, "b12x_mxfp8_packed_weight", None), + packed_weight, + ) ++ layer.b12x_activation_mode = get_b12x_dense_activation_mode("mxfp8") ++ layer.b12x_bf16_input_supported = ( ++ current_platform.is_device_capability_family(120) ++ and in_features % 128 == 0 ++ and out_features % 8 == 0 ++ ) + replace_parameter(layer, "weight", weight.new_empty((0,))) + replace_parameter(layer, "weight_scale", weight_scale.new_empty((0,))) + layer.b12x_warmup_provider = self +@@ -114,20 +138,13 @@ class B12xMxfp8LinearKernel(Mxfp8LinearKernel): + device = torch.device(packed_weight.weight.values.device) + + def compile() -> None: +- mxfp8 = _import_b12x_mxfp8() +- assert mxfp8 is not None + for tokens in token_counts: + source = torch.zeros( + (tokens, int(packed_weight.in_features)), + dtype=output_dtype, + device=device, + ) +- mxfp8.mm( +- source, +- packed_weight, +- expected_m=max(1, int(tokens)), +- stream=current_stream().cuda_stream, +- ) ++ _apply_b12x_mxfp8_packed_linear(layer, source, None) + + return B12xWarmupUnit( + name="MXFP8", +@@ -137,6 +154,8 @@ class B12xMxfp8LinearKernel(Mxfp8LinearKernel): + int(packed_weight.in_features), + int(packed_weight.padded_in_features), + int(packed_weight.out_features), ++ layer.b12x_activation_mode, ++ layer.b12x_bf16_input_supported, + output_dtype, + ), + compile=compile, +diff --git a/vllm/model_executor/kernels/linear/nvfp4/b12x.py b/vllm/model_executor/kernels/linear/nvfp4/b12x.py +index 4ff531dd48be9ec7808872847d9665e1b893d7a6..aee238b3f30fb5a871f3b212b58dcd37916c0fed 100644 +--- a/vllm/model_executor/kernels/linear/nvfp4/b12x.py ++++ b/vllm/model_executor/kernels/linear/nvfp4/b12x.py +@@ -8,7 +8,7 @@ import torch + from vllm._custom_ops import scaled_fp4_quant + from vllm.model_executor.utils import replace_parameter + from vllm.platforms import current_platform +-from vllm.utils.b12x import B12xWarmupUnit ++from vllm.utils.b12x import B12xWarmupUnit, get_b12x_dense_activation_mode + from vllm.utils.b12x import ( + get_b12x_blockscaled as _import_b12x_blockscaled, + ) +@@ -18,30 +18,40 @@ from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig + + + def _apply_b12x_nvfp4_linear( ++ layer: torch.nn.Module, + x: torch.Tensor, +- weight: torch.Tensor, +- weight_scale_storage: torch.Tensor, +- input_global_scale_inv: torch.Tensor, +- alpha: torch.Tensor, + bias: torch.Tensor | None, + ) -> torch.Tensor: + blockscaled = _import_b12x_blockscaled() + assert blockscaled is not None + +- output_size = int(weight.shape[0]) ++ output_size = int(layer.weight.shape[0]) + output_shape = [*x.shape[:-1], output_size] + x_2d = x.reshape(-1, x.shape[-1]) ++ mode = layer.b12x_activation_mode ++ if x.dtype == torch.bfloat16 and layer.b12x_bf16_input_supported: ++ output = blockscaled.mm( ++ x_2d.contiguous(), ++ layer.b12x_nvfp4_packed_weight, ++ mode=mode, ++ activation_global_scale=layer.input_global_scale_inv, ++ bias=bias, ++ expected_m=max(1, int(x_2d.shape[0])), ++ ) ++ return output.view(*output_shape) ++ if mode == "a16": ++ raise ValueError("b12x NVFP4 A16 requires BF16 on SM120/SM121 with K%128=N%8=0") + x_packed, x_scale_swizzled = scaled_fp4_quant( + x_2d, +- input_global_scale_inv, ++ layer.input_global_scale_inv, + is_sf_swizzled_layout=True, + ) + output = blockscaled.mm_nvfp4( + x_packed, + x_scale_swizzled, +- weight, +- weight_scale_storage, +- alpha, ++ layer.weight, ++ layer.weight_scale, ++ layer.alpha, + out_dtype=x.dtype, + ) + if bias is not None: +@@ -66,6 +76,11 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel): + return False, "Install the B12X backend with `pip install vllm[b12x]`" + if not blockscaled.is_supported(): + return False, "b12x native NVFP4 GEMM is not supported" ++ if not hasattr(blockscaled, "w4a16"): ++ return ( ++ False, ++ "b12x NVFP4 requires a source build with dense precision selection", ++ ) + return True, None + + @classmethod +@@ -81,6 +96,21 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel): + "weight_scale", + intrinsics.swizzle_block_scale(layer.weight_scale.data), + ) ++ blockscaled = _import_b12x_blockscaled() ++ assert blockscaled is not None ++ layer.b12x_nvfp4_packed_weight = blockscaled.pack_weight( ++ layer.weight.data, ++ layer.weight_scale.data, ++ recipe="nvfp4", ++ global_scale=layer.weight_global_scale, ++ ) ++ layer.b12x_activation_mode = get_b12x_dense_activation_mode("nvfp4") ++ n, packed_k = layer.weight.shape ++ layer.b12x_bf16_input_supported = ( ++ current_platform.is_device_capability_family(120) ++ and (packed_k * 2) % 128 == 0 ++ and n % 8 == 0 ++ ) + layer.b12x_warmup_provider = self + + def get_b12x_warmup_unit( +@@ -100,11 +130,8 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel): + (tokens, k), dtype=output_dtype, device=weight.device + ) + _apply_b12x_nvfp4_linear( ++ layer, + source, +- weight, +- weight_scale, +- layer.input_global_scale_inv, +- layer.alpha, + None, + ) + +@@ -117,6 +144,8 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel): + k, + weight.dtype, + weight_scale.dtype, ++ layer.b12x_activation_mode, ++ layer.b12x_bf16_input_supported, + output_dtype, + ), + compile=compile, +@@ -129,11 +158,8 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel): + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + return _apply_b12x_nvfp4_linear( ++ layer, + x, +- layer.weight, +- layer.weight_scale, +- layer.input_global_scale_inv, +- layer.alpha, + bias, + ) + +diff --git a/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py +index aacfb1dc59bac0507c4ad2b743b3be1137ad054e..9fe6ab86c26d4685fb07d9407ea67fad1e01e872 100644 +--- a/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py ++++ b/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py +@@ -991,8 +991,7 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): + """Execute B12X KDA after the convolution projection. + + Args: +- metadata: Forward-context metadata used to share runtime-owned +- packed metadata tensors across compatible layers. ++ metadata: Describes whether packed query boundaries are uniform. + mixed_qkv: Live packed query, key, and value projection. + raw_g: Live unactivated forget gate. + raw_beta: Live unactivated update gate. +@@ -1032,17 +1031,16 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): + cache = forward_context.additional_kwargs.setdefault( + "b12x_kda_metadata_tensors", {} + ) ++ # Uniform builders own separate buffers with identical fixed boundaries. + cache_key = ( +- id(metadata), ++ None if metadata.is_uniform_spec_decode else query_start_loc.data_ptr(), ++ num_accepted_tokens.data_ptr() if num_accepted_tokens is not None else None, + num_tokens, + num_requests, +- state_columns, +- plan.caps.max_state_slots, + ) + bound_metadata = cache.get(cache_key) + if bound_metadata is None: + query_start_loc = query_start_loc[: num_requests + 1] +- state_indices = state_indices[:num_requests, :state_columns] + if num_accepted_tokens is None: + accepted_tokens = self._b12x_kda_num_accepted_tokens[:num_requests] + accepted_tokens.fill_(1) +@@ -1055,7 +1053,6 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): + bound_metadata = ( + query_start_loc, + accepted_tokens, +- state_indices, + self._b12x_kda_num_seqs, + self._b12x_kda_num_tokens, + ) +@@ -1063,7 +1060,6 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): + ( + query_start_loc, + accepted_tokens, +- state_indices, + num_seqs, + num_tokens_tensor, + ) = bound_metadata +@@ -1081,7 +1077,7 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): + recurrent_state=self.kv_cache[1], + query_start_loc=query_start_loc, + num_accepted_tokens=accepted_tokens, +- state_indices=state_indices, ++ state_indices=state_indices[:num_requests, :state_columns], + num_seqs=num_seqs, + num_tokens=num_tokens_tensor, + output=output, +diff --git a/vllm/model_executor/layers/quantization/online/nvfp4.py b/vllm/model_executor/layers/quantization/online/nvfp4.py +index de920db901aa695d2a0eb3798c4ae67d64bd181c..3d36f872b0fb87482cdc48051ff7d3adeab959d7 100644 +--- a/vllm/model_executor/layers/quantization/online/nvfp4.py ++++ b/vllm/model_executor/layers/quantization/online/nvfp4.py +@@ -1,192 +1,261 @@ +-# SPDX-License-Identifier: Apache-2.0 +-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +- +-import torch ++# SPDX-License-Identifier: Apache-2.0 ++# SPDX-FileCopyrightText: Copyright contributors to the vLLM project ++ ++import torch + from torch.nn import Module + + from vllm._custom_ops import scaled_fp4_quant +-from vllm.model_executor.layers.fused_moe import RoutedExperts +-from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig +-from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( +- convert_to_nvfp4_moe_kernel_format, +- make_nvfp4_moe_kernel, +- make_nvfp4_moe_quant_config, +- select_nvfp4_moe_backend, ++from vllm.model_executor.kernels.linear.nvfp4.b12x import ( ++ B12xNvFp4LinearKernel, + ) ++from vllm.model_executor.kernels.linear.nvfp4.base import NvFp4LinearLayerConfig ++from vllm.model_executor.layers.fused_moe import RoutedExperts ++from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig ++from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( ++ convert_to_nvfp4_moe_kernel_format, ++ make_nvfp4_moe_kernel, ++ make_nvfp4_moe_quant_config, ++ select_nvfp4_moe_backend, ++) ++from vllm.model_executor.layers.quantization.online.fp8 import _Fp8OnlineLinearBase + from vllm.model_executor.layers.quantization.online.moe_base import ( + OnlineMoEMethodBase, + ) +-from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( +- FLOAT4_E2M1_MAX, +-) +-from vllm.model_executor.layers.quantization.utils.quant_utils import ( +- amax_for_moe_weight_quant, +- kNvfp4Dynamic, +- kNvfp4Static, +- weight_amax, +-) +-from vllm.model_executor.utils import replace_parameter ++from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( ++ FLOAT4_E2M1_MAX, ++) ++from vllm.model_executor.layers.quantization.utils.quant_utils import ( ++ amax_for_moe_weight_quant, ++ kNvfp4Dynamic, ++ kNvfp4Static, ++ weight_amax, ++) ++from vllm.model_executor.utils import replace_parameter + from vllm.platforms import current_platform +- ++from vllm.utils.b12x import get_b12x_blockscaled ++ + FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max + + +-def _quantize_moe_weight_to_nvfp4( +- weight: torch.Tensor, +- moe_tp_size: int = 1, +-) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: +- """Quantize stacked MoE expert weights ``(E, N, K)`` to NVFP4. +- +- One FP32 global scale per expert plus per-block (group-16) FP8 scales, +- matching the ModelOpt NVFP4 checkpoint layout. Returns packed FP4 weights +- ``(E, N, K // 2)``, block scales ``(E, N, K // 16)``, and the per-expert +- global scale ``(E,)`` stored as ``amax / (fp4_max * fp8_max)``. +- """ +- assert weight.dim() == 3, f"expected 3D expert weights, got {weight.shape}" +- k = weight.shape[-1] +- assert k % 16 == 0, f"last dim must be a multiple of 16, got {k}" +- +- amax = weight_amax(weight.flatten(1), dim=-1).to(torch.float32) +- amax = amax_for_moe_weight_quant(amax, moe_tp_size).clamp_min(1e-8) +- global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax +- weight_scale_2 = (1.0 / global_scale).to(torch.float32) ++class Nvfp4OnlineLinearMethod(_Fp8OnlineLinearBase): ++ """Load a BF16 linear weight as NVFP4 for the proposal head.""" + +- # Keep the original BF16/FP16 values as the quantizer input. Folding each +- # expert's FP32 global scale into the weight would add a BF16/FP16 rounding +- # before the group-16 scale and E2M1 values are selected. +- weight = weight.contiguous() +- quantized_experts = [ +- scaled_fp4_quant( +- expert_weight, +- expert_scale, +- is_sf_swizzled_layout=False, +- ) +- for expert_weight, expert_scale in zip( +- weight, +- global_scale, +- strict=True, +- ) +- ] +- qweight = torch.stack([quantized for quantized, _ in quantized_experts]) +- block_scale = torch.stack([block_scale for _, block_scale in quantized_experts]) +- return ( +- qweight, +- block_scale, +- weight_scale_2, +- ) +- +- +-class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase): +- """Online NVFP4 MoE quantization with per-token activation scales. ++ def __init__(self, *, use_a16: bool = False): ++ super().__init__() ++ supported, reason = B12xNvFp4LinearKernel.is_supported() ++ if not supported: ++ raise ValueError(f"Online NVFP4 proposal head requires b12x: {reason}") ++ self.kernel = B12xNvFp4LinearKernel(NvFp4LinearLayerConfig()) ++ self.use_a16 = use_a16 ++ if use_a16 and self.input_dtype != torch.bfloat16: ++ raise ValueError("A16 proposal heads require BF16 activations") + +- Quantizes fp16/bf16 expert weights to NVFP4 at load time; the FlashInfer +- TRTLLM kernel computes per-token activation scales at runtime. Blackwell +- (SM100) only. +- """ +- +- def __init__( +- self, +- *, +- layer: torch.nn.Module, +- ): +- if not current_platform.is_device_capability_family(100): +- raise ValueError( +- "nvfp4_per_token online quantization requires a Blackwell (SM100) GPU." +- ) +- super().__init__(layer.moe_config) +- self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend( +- config=self.moe, +- weight_key=kNvfp4Static, +- activation_key=kNvfp4Dynamic, +- ) +- +- def process_weights_after_loading(self, layer: Module) -> None: ++ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if getattr(layer, "_already_called_process_weights_after_loading", False): + return +- +- self._quantize_weights(layer) +- self._setup_kernel(layer) +- +- layer._already_called_process_weights_after_loading = True +- +- def _quantize_weights(self, layer: Module) -> None: +- moe_tp_size = self.moe.tp_size +- w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4( +- layer.w13_weight, moe_tp_size +- ) +- w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4( +- layer.w2_weight, moe_tp_size ++ weight = layer.weight.contiguous() ++ if weight.shape[1] % 16: ++ raise ValueError("Online NVFP4 proposal head requires K divisible by 16") ++ amax = weight.abs().amax().float().clamp_min(1e-8) ++ global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax ++ packed, scales = scaled_fp4_quant( ++ weight, global_scale, is_sf_swizzled_layout=False + ) ++ replace_parameter(layer, "weight", packed) ++ replace_parameter(layer, "weight_scale", scales) ++ replace_parameter(layer, "weight_global_scale", global_scale.reciprocal()) ++ replace_parameter(layer, "input_global_scale_inv", torch.ones_like(amax)) ++ replace_parameter(layer, "alpha", layer.weight_global_scale.clone()) ++ self.kernel.process_weights_after_loading(layer) ++ layer.b12x_activation_mode = "a16" if self.use_a16 else "quantized" ++ layer._already_called_process_weights_after_loading = True + +- replace_parameter(layer, "w13_weight", w13) +- replace_parameter(layer, "w13_weight_scale", w13_scale) +- replace_parameter(layer, "w13_weight_scale_2", w13_scale_2) +- replace_parameter(layer, "w2_weight", w2) +- replace_parameter(layer, "w2_weight_scale", w2_scale) +- replace_parameter(layer, "w2_weight_scale_2", w2_scale_2) +- +- # Neutral (1.0) activation global scales: the kernel derives per-token +- # scales at runtime, so the output scalars reduce to the weight scales. +- ones = torch.ones(layer.num_experts, device=w13.device, dtype=torch.float32) +- replace_parameter(layer, "w13_input_scale", ones) +- replace_parameter(layer, "w2_input_scale", ones.clone()) +- +- def _setup_kernel(self, layer: RoutedExperts) -> None: +- ( +- w13, +- w13_scale, +- w13_scale_2, +- a13_scale, +- w2, +- w2_scale, +- w2_scale_2, +- a2_scale, +- ) = convert_to_nvfp4_moe_kernel_format( +- nvfp4_backend=self.nvfp4_backend, +- layer=layer, +- w13=layer.w13_weight, +- w13_scale=layer.w13_weight_scale, +- w13_scale_2=layer.w13_weight_scale_2, +- a13_scale=layer.w13_input_scale, +- w2=layer.w2_weight, +- w2_scale=layer.w2_weight_scale, +- w2_scale_2=layer.w2_weight_scale_2, +- a2_scale=layer.w2_input_scale, +- is_act_and_mul=self.moe.is_act_and_mul, ++ def apply( ++ self, ++ layer: torch.nn.Module, ++ x: torch.Tensor, ++ bias: torch.Tensor | None = None, ++ ) -> torch.Tensor: ++ if self.use_a16: ++ return self.kernel.apply_weights(layer, x, bias) ++ amax = x.abs().amax().float().clamp_min(1e-8) ++ input_scale = amax / (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) ++ x_packed, x_scale = scaled_fp4_quant( ++ x.reshape(-1, x.shape[-1]), ++ input_scale.reciprocal(), ++ is_sf_swizzled_layout=True, + ) +- +- replace_parameter(layer, "w13_weight", w13) +- replace_parameter(layer, "w13_weight_scale", w13_scale) +- replace_parameter(layer, "w13_weight_scale_2", w13_scale_2) +- replace_parameter(layer, "w13_input_scale", a13_scale) +- replace_parameter(layer, "w2_weight", w2) +- replace_parameter(layer, "w2_weight_scale", w2_scale) +- replace_parameter(layer, "w2_weight_scale_2", w2_scale_2) +- replace_parameter(layer, "w2_input_scale", a2_scale) +- +- if self.moe_kernel is None: +- self.moe_quant_config = self.get_fused_moe_quant_config(layer) +- assert self.experts_cls is not None +- self.moe_kernel = make_nvfp4_moe_kernel( +- moe_quant_config=self.moe_quant_config, +- moe_config=self.moe, +- experts_cls=self.experts_cls, +- backend=self.nvfp4_backend, +- routing_tables=layer._expert_routing_tables(), +- per_token_activation=True, +- ) +- +- self.moe_kernel.fused_experts.process_weights_after_loading(layer) +- +- def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: +- return make_nvfp4_moe_quant_config( +- backend=self.nvfp4_backend, +- w13_scale=layer.w13_weight_scale, +- w2_scale=layer.w2_weight_scale, +- w13_scale_2=layer.w13_weight_scale_2, +- w2_scale_2=layer.w2_weight_scale_2, +- a13_scale=layer.w13_input_scale, +- a2_scale=layer.w2_input_scale, +- swiglu_limit=getattr(layer, "swiglu_limit", None), +- layer=layer, ++ blockscaled = get_b12x_blockscaled() ++ assert blockscaled is not None ++ output = blockscaled.mm_nvfp4( ++ x_packed, ++ x_scale, ++ layer.weight, ++ layer.weight_scale, ++ input_scale * layer.weight_global_scale, ++ out_dtype=x.dtype, + ) ++ if bias is not None: ++ output = output + bias ++ return output.view(*x.shape[:-1], layer.weight.shape[0]) ++ ++ ++def _quantize_moe_weight_to_nvfp4( ++ weight: torch.Tensor, ++ moe_tp_size: int = 1, ++) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: ++ """Quantize stacked MoE expert weights ``(E, N, K)`` to NVFP4. ++ ++ One FP32 global scale per expert plus per-block (group-16) FP8 scales, ++ matching the ModelOpt NVFP4 checkpoint layout. Returns packed FP4 weights ++ ``(E, N, K // 2)``, block scales ``(E, N, K // 16)``, and the per-expert ++ global scale ``(E,)`` stored as ``amax / (fp4_max * fp8_max)``. ++ """ ++ assert weight.dim() == 3, f"expected 3D expert weights, got {weight.shape}" ++ k = weight.shape[-1] ++ assert k % 16 == 0, f"last dim must be a multiple of 16, got {k}" ++ ++ amax = weight_amax(weight.flatten(1), dim=-1).to(torch.float32) ++ amax = amax_for_moe_weight_quant(amax, moe_tp_size).clamp_min(1e-8) ++ global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax ++ weight_scale_2 = (1.0 / global_scale).to(torch.float32) ++ ++ # Keep the original BF16/FP16 values as the quantizer input. Folding each ++ # expert's FP32 global scale into the weight would add a BF16/FP16 rounding ++ # before the group-16 scale and E2M1 values are selected. ++ weight = weight.contiguous() ++ quantized_experts = [ ++ scaled_fp4_quant( ++ expert_weight, ++ expert_scale, ++ is_sf_swizzled_layout=False, ++ ) ++ for expert_weight, expert_scale in zip( ++ weight, ++ global_scale, ++ strict=True, ++ ) ++ ] ++ qweight = torch.stack([quantized for quantized, _ in quantized_experts]) ++ block_scale = torch.stack([block_scale for _, block_scale in quantized_experts]) ++ return ( ++ qweight, ++ block_scale, ++ weight_scale_2, ++ ) ++ ++ ++class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase): ++ """Online NVFP4 MoE quantization with per-token activation scales. ++ ++ Quantizes fp16/bf16 expert weights to NVFP4 at load time; the FlashInfer ++ TRTLLM kernel computes per-token activation scales at runtime. Blackwell ++ (SM100) only. ++ """ ++ ++ def __init__( ++ self, ++ *, ++ layer: torch.nn.Module, ++ ): ++ if not current_platform.is_device_capability_family(100): ++ raise ValueError( ++ "nvfp4_per_token online quantization requires a Blackwell (SM100) GPU." ++ ) ++ super().__init__(layer.moe_config) ++ self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend( ++ config=self.moe, ++ weight_key=kNvfp4Static, ++ activation_key=kNvfp4Dynamic, ++ ) ++ ++ def process_weights_after_loading(self, layer: Module) -> None: ++ if getattr(layer, "_already_called_process_weights_after_loading", False): ++ return ++ ++ self._quantize_weights(layer) ++ self._setup_kernel(layer) ++ ++ layer._already_called_process_weights_after_loading = True ++ ++ def _quantize_weights(self, layer: Module) -> None: ++ moe_tp_size = self.moe.tp_size ++ w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4( ++ layer.w13_weight, moe_tp_size ++ ) ++ w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4( ++ layer.w2_weight, moe_tp_size ++ ) ++ ++ replace_parameter(layer, "w13_weight", w13) ++ replace_parameter(layer, "w13_weight_scale", w13_scale) ++ replace_parameter(layer, "w13_weight_scale_2", w13_scale_2) ++ replace_parameter(layer, "w2_weight", w2) ++ replace_parameter(layer, "w2_weight_scale", w2_scale) ++ replace_parameter(layer, "w2_weight_scale_2", w2_scale_2) ++ ++ # Neutral (1.0) activation global scales: the kernel derives per-token ++ # scales at runtime, so the output scalars reduce to the weight scales. ++ ones = torch.ones(layer.num_experts, device=w13.device, dtype=torch.float32) ++ replace_parameter(layer, "w13_input_scale", ones) ++ replace_parameter(layer, "w2_input_scale", ones.clone()) ++ ++ def _setup_kernel(self, layer: RoutedExperts) -> None: ++ ( ++ w13, ++ w13_scale, ++ w13_scale_2, ++ a13_scale, ++ w2, ++ w2_scale, ++ w2_scale_2, ++ a2_scale, ++ ) = convert_to_nvfp4_moe_kernel_format( ++ nvfp4_backend=self.nvfp4_backend, ++ layer=layer, ++ w13=layer.w13_weight, ++ w13_scale=layer.w13_weight_scale, ++ w13_scale_2=layer.w13_weight_scale_2, ++ a13_scale=layer.w13_input_scale, ++ w2=layer.w2_weight, ++ w2_scale=layer.w2_weight_scale, ++ w2_scale_2=layer.w2_weight_scale_2, ++ a2_scale=layer.w2_input_scale, ++ is_act_and_mul=self.moe.is_act_and_mul, ++ ) ++ ++ replace_parameter(layer, "w13_weight", w13) ++ replace_parameter(layer, "w13_weight_scale", w13_scale) ++ replace_parameter(layer, "w13_weight_scale_2", w13_scale_2) ++ replace_parameter(layer, "w13_input_scale", a13_scale) ++ replace_parameter(layer, "w2_weight", w2) ++ replace_parameter(layer, "w2_weight_scale", w2_scale) ++ replace_parameter(layer, "w2_weight_scale_2", w2_scale_2) ++ replace_parameter(layer, "w2_input_scale", a2_scale) ++ ++ if self.moe_kernel is None: ++ self.moe_quant_config = self.get_fused_moe_quant_config(layer) ++ assert self.experts_cls is not None ++ self.moe_kernel = make_nvfp4_moe_kernel( ++ moe_quant_config=self.moe_quant_config, ++ moe_config=self.moe, ++ experts_cls=self.experts_cls, ++ backend=self.nvfp4_backend, ++ routing_tables=layer._expert_routing_tables(), ++ per_token_activation=True, ++ ) ++ ++ self.moe_kernel.fused_experts.process_weights_after_loading(layer) ++ ++ def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: ++ return make_nvfp4_moe_quant_config( ++ backend=self.nvfp4_backend, ++ w13_scale=layer.w13_weight_scale, ++ w2_scale=layer.w2_weight_scale, ++ w13_scale_2=layer.w13_weight_scale_2, ++ w2_scale_2=layer.w2_weight_scale_2, ++ a13_scale=layer.w13_input_scale, ++ a2_scale=layer.w2_input_scale, ++ swiglu_limit=getattr(layer, "swiglu_limit", None), ++ layer=layer, ++ ) +diff --git a/vllm/model_executor/layers/vocab_parallel_embedding.py b/vllm/model_executor/layers/vocab_parallel_embedding.py +index 3057270c62185e767cfd48f63593c2705a1b1b79..2adf622d9372d36013e16ba591738ecbc78d77c0 100644 +--- a/vllm/model_executor/layers/vocab_parallel_embedding.py ++++ b/vllm/model_executor/layers/vocab_parallel_embedding.py +@@ -1,581 +1,604 @@ +-# SPDX-License-Identifier: Apache-2.0 +-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +- ++# SPDX-License-Identifier: Apache-2.0 ++# SPDX-FileCopyrightText: Copyright contributors to the vLLM project ++ + from collections.abc import Sequence + from dataclasses import dataclass +- +-import torch +-import torch.nn.functional as F +-from torch.nn.parameter import Parameter +- +-import vllm.envs as envs +-from vllm.distributed import ( +- divide, +- get_tensor_model_parallel_rank, +- get_tensor_model_parallel_world_size, +- tensor_model_parallel_all_reduce, +-) +-from vllm.model_executor.custom_op import PluggableLayer +-from vllm.model_executor.determinism.batch_invariant import ( +- linear_batch_invariant, +-) +-from vllm.model_executor.layers.quantization.base_config import ( +- QuantizationConfig, +- QuantizeMethodBase, +- method_has_implemented_embedding, +-) +-from vllm.model_executor.layers.utils import dispatch_unquantized_gemm +-from vllm.model_executor.parameter import BasevLLMParameter +-from vllm.model_executor.utils import set_weight_attrs +-from vllm.platforms import current_platform +- +-DEFAULT_VOCAB_PADDING_SIZE = 64 +- +- +-class UnquantizedEmbeddingMethod(QuantizeMethodBase): +- """Unquantized method for embeddings.""" +- +- def create_weights( +- self, +- layer: torch.nn.Module, +- input_size_per_partition: int, +- output_partition_sizes: list[int], +- input_size: int, +- output_size: int, +- params_dtype: torch.dtype, +- **extra_weight_attrs, +- ): +- """Create weights for embedding layer.""" +- weight = Parameter( +- torch.empty( +- sum(output_partition_sizes), +- input_size_per_partition, +- dtype=params_dtype, +- ), +- requires_grad=False, +- ) +- set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) +- layer.register_parameter("weight", weight) +- set_weight_attrs(weight, extra_weight_attrs) +- +- def process_weights_after_loading(self, layer: torch.nn.Module) -> None: +- if current_platform.is_cpu(): +- from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm +- +- dispatch_cpu_unquantized_gemm(layer, remove_weight=False) +- +- def apply( +- self, +- layer: torch.nn.Module, +- x: torch.Tensor, +- bias: torch.Tensor | None = None, +- ) -> torch.Tensor: +- if envs.VLLM_BATCH_INVARIANT and current_platform.is_cuda_alike(): +- return linear_batch_invariant(x, layer.weight, bias) +- return dispatch_unquantized_gemm()(layer, x, layer.weight, bias) +- +- def embedding(self, layer: torch.nn.Module, input_: torch.Tensor) -> torch.Tensor: +- return F.embedding(input_, layer.weight) +- +- def tie_weights( +- self, layer: torch.nn.Module, embed_tokens: "VocabParallelEmbedding" +- ): +- layer.weight = embed_tokens.weight +- return layer +- +- +-def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int: +- """Pad the vocab size to the given value.""" +- return ((vocab_size + pad_to - 1) // pad_to) * pad_to +- +- +-def vocab_range_from_per_partition_vocab_size( +- per_partition_vocab_size: int, rank: int, offset: int = 0 +-) -> Sequence[int]: +- index_f = rank * per_partition_vocab_size +- index_l = index_f + per_partition_vocab_size +- return index_f + offset, index_l + offset +- +- +-def vocab_range_from_global_vocab_size( +- global_vocab_size: int, rank: int, world_size: int, offset: int = 0 +-) -> Sequence[int]: +- per_partition_vocab_size = divide(global_vocab_size, world_size) +- return vocab_range_from_per_partition_vocab_size( +- per_partition_vocab_size, rank, offset=offset +- ) +- +- +-@dataclass +-class VocabParallelEmbeddingShardIndices: +- """Indices for a shard of a vocab parallel embedding.""" +- +- padded_org_vocab_start_index: int +- padded_org_vocab_end_index: int +- padded_added_vocab_start_index: int +- padded_added_vocab_end_index: int +- +- org_vocab_start_index: int +- org_vocab_end_index: int +- added_vocab_start_index: int +- added_vocab_end_index: int +- +- @property +- def num_org_elements(self) -> int: +- return self.org_vocab_end_index - self.org_vocab_start_index +- +- @property +- def num_added_elements(self) -> int: +- return self.added_vocab_end_index - self.added_vocab_start_index +- +- @property +- def num_org_elements_padded(self) -> int: +- return self.padded_org_vocab_end_index - self.padded_org_vocab_start_index +- +- @property +- def num_added_elements_padded(self) -> int: +- return self.padded_added_vocab_end_index - self.padded_added_vocab_start_index +- +- @property +- def num_org_vocab_padding(self) -> int: +- return self.num_org_elements_padded - self.num_org_elements +- +- @property +- def num_added_vocab_padding(self) -> int: +- return self.num_added_elements_padded - self.num_added_elements +- +- @property +- def num_elements_padded(self) -> int: +- return self.num_org_elements_padded + self.num_added_elements_padded +- +- def __post_init__(self): +- # sanity checks +- assert self.padded_org_vocab_start_index <= self.padded_org_vocab_end_index +- assert self.padded_added_vocab_start_index <= self.padded_added_vocab_end_index +- +- assert self.org_vocab_start_index <= self.org_vocab_end_index +- assert self.added_vocab_start_index <= self.added_vocab_end_index +- +- assert self.org_vocab_start_index <= self.padded_org_vocab_start_index +- assert self.added_vocab_start_index <= self.padded_added_vocab_start_index +- assert self.org_vocab_end_index <= self.padded_org_vocab_end_index +- assert self.added_vocab_end_index <= self.padded_added_vocab_end_index +- +- assert self.num_org_elements <= self.num_org_elements_padded +- assert self.num_added_elements <= self.num_added_elements_padded +- +- +-@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend) +-def get_masked_input_and_mask( +- input_: torch.Tensor, +- org_vocab_start_index: int, +- org_vocab_end_index: int, +- num_org_vocab_padding: int, +- added_vocab_start_index: int, +- added_vocab_end_index: int, +-) -> tuple[torch.Tensor, torch.Tensor]: +- # torch.compile will fuse all of the pointwise ops below +- # into a single kernel, making it very fast +- org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ < org_vocab_end_index) +- added_vocab_mask = (input_ >= added_vocab_start_index) & ( +- input_ < added_vocab_end_index +- ) +- added_offset = ( +- added_vocab_start_index +- - (org_vocab_end_index - org_vocab_start_index) +- - num_org_vocab_padding +- ) +- valid_offset = (org_vocab_start_index * org_vocab_mask) + ( +- added_offset * added_vocab_mask +- ) +- vocab_mask = org_vocab_mask | added_vocab_mask +- input_ = vocab_mask * (input_ - valid_offset) +- return input_, ~vocab_mask +- +- +-# --8<-- [start:vocab_parallel_embedding] +-@PluggableLayer.register("vocab_parallel_embedding") +-class VocabParallelEmbedding(PluggableLayer): +- """Embedding parallelized in the vocabulary dimension. +- +- Adapted from torch.nn.Embedding, note that we pad the vocabulary size to +- make sure it is divisible by the number of model parallel GPUs. +- +- In order to support various loading methods, we ensure that LoRA-added +- embeddings are always at the end of TP-sharded tensors. In other words, +- we shard base embeddings and LoRA embeddings separately (both padded), +- and place them in the same tensor. +- In this example, we will have the original vocab size = 1010, +- added vocab size = 16 and padding to 64. Therefore, the total +- vocab size with padding will be 1088 (because we first pad 1010 to +- 1024, add 16, and then pad to 1088). +- Therefore, the tensor format looks like the following: +- TP1, rank 0 (no sharding): +- |< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >| +- corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1025 | -1 | ... | -1 | +- index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 | +- +- TP2, rank 0: +- |< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >| +- corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1010 | ... | 1025 | -1 | ... | -1 | +- index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 | +- TP2, rank 1: +- |< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >| +- corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 | +- index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 | +- +- Args: +- num_embeddings: vocabulary size. +- embedding_dim: size of hidden state. +- params_dtype: type of the parameters. +- org_num_embeddings: original vocabulary size (without LoRA). +- padding_size: padding size for the vocabulary. +- quant_config: quant config for the layer +- prefix: full name of the layer in the state dict +- disable_tp: If true, tensor parallelism will be disabled for this layer. +- """ # noqa: E501 +- +- # --8<-- [end:vocab_parallel_embedding] +- +- def __init__( +- self, +- num_embeddings: int, +- embedding_dim: int, +- params_dtype: torch.dtype | None = None, +- org_num_embeddings: int | None = None, +- padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, +- quant_config: QuantizationConfig | None = None, +- prefix: str = "", ++from typing import Literal ++ ++import torch ++import torch.nn.functional as F ++from torch.nn.parameter import Parameter ++ ++import vllm.envs as envs ++from vllm.distributed import ( ++ divide, ++ get_tensor_model_parallel_rank, ++ get_tensor_model_parallel_world_size, ++ tensor_model_parallel_all_reduce, ++) ++from vllm.model_executor.custom_op import PluggableLayer ++from vllm.model_executor.determinism.batch_invariant import ( ++ linear_batch_invariant, ++) ++from vllm.model_executor.layers.quantization.base_config import ( ++ QuantizationConfig, ++ QuantizeMethodBase, ++ method_has_implemented_embedding, ++) ++from vllm.model_executor.layers.utils import dispatch_unquantized_gemm ++from vllm.model_executor.parameter import BasevLLMParameter ++from vllm.model_executor.utils import set_weight_attrs ++from vllm.platforms import current_platform ++ ++DEFAULT_VOCAB_PADDING_SIZE = 64 ++ ++ ++class UnquantizedEmbeddingMethod(QuantizeMethodBase): ++ """Unquantized method for embeddings.""" ++ ++ def create_weights( ++ self, ++ layer: torch.nn.Module, ++ input_size_per_partition: int, ++ output_partition_sizes: list[int], ++ input_size: int, ++ output_size: int, ++ params_dtype: torch.dtype, ++ **extra_weight_attrs, ++ ): ++ """Create weights for embedding layer.""" ++ weight = Parameter( ++ torch.empty( ++ sum(output_partition_sizes), ++ input_size_per_partition, ++ dtype=params_dtype, ++ ), ++ requires_grad=False, ++ ) ++ set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) ++ layer.register_parameter("weight", weight) ++ set_weight_attrs(weight, extra_weight_attrs) ++ ++ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: ++ if current_platform.is_cpu(): ++ from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm ++ ++ dispatch_cpu_unquantized_gemm(layer, remove_weight=False) ++ ++ def apply( ++ self, ++ layer: torch.nn.Module, ++ x: torch.Tensor, ++ bias: torch.Tensor | None = None, ++ ) -> torch.Tensor: ++ if envs.VLLM_BATCH_INVARIANT and current_platform.is_cuda_alike(): ++ return linear_batch_invariant(x, layer.weight, bias) ++ return dispatch_unquantized_gemm()(layer, x, layer.weight, bias) ++ ++ def embedding(self, layer: torch.nn.Module, input_: torch.Tensor) -> torch.Tensor: ++ return F.embedding(input_, layer.weight) ++ ++ def tie_weights( ++ self, layer: torch.nn.Module, embed_tokens: "VocabParallelEmbedding" ++ ): ++ layer.weight = embed_tokens.weight ++ return layer ++ ++ ++def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int: ++ """Pad the vocab size to the given value.""" ++ return ((vocab_size + pad_to - 1) // pad_to) * pad_to ++ ++ ++def vocab_range_from_per_partition_vocab_size( ++ per_partition_vocab_size: int, rank: int, offset: int = 0 ++) -> Sequence[int]: ++ index_f = rank * per_partition_vocab_size ++ index_l = index_f + per_partition_vocab_size ++ return index_f + offset, index_l + offset ++ ++ ++def vocab_range_from_global_vocab_size( ++ global_vocab_size: int, rank: int, world_size: int, offset: int = 0 ++) -> Sequence[int]: ++ per_partition_vocab_size = divide(global_vocab_size, world_size) ++ return vocab_range_from_per_partition_vocab_size( ++ per_partition_vocab_size, rank, offset=offset ++ ) ++ ++ ++@dataclass ++class VocabParallelEmbeddingShardIndices: ++ """Indices for a shard of a vocab parallel embedding.""" ++ ++ padded_org_vocab_start_index: int ++ padded_org_vocab_end_index: int ++ padded_added_vocab_start_index: int ++ padded_added_vocab_end_index: int ++ ++ org_vocab_start_index: int ++ org_vocab_end_index: int ++ added_vocab_start_index: int ++ added_vocab_end_index: int ++ ++ @property ++ def num_org_elements(self) -> int: ++ return self.org_vocab_end_index - self.org_vocab_start_index ++ ++ @property ++ def num_added_elements(self) -> int: ++ return self.added_vocab_end_index - self.added_vocab_start_index ++ ++ @property ++ def num_org_elements_padded(self) -> int: ++ return self.padded_org_vocab_end_index - self.padded_org_vocab_start_index ++ ++ @property ++ def num_added_elements_padded(self) -> int: ++ return self.padded_added_vocab_end_index - self.padded_added_vocab_start_index ++ ++ @property ++ def num_org_vocab_padding(self) -> int: ++ return self.num_org_elements_padded - self.num_org_elements ++ ++ @property ++ def num_added_vocab_padding(self) -> int: ++ return self.num_added_elements_padded - self.num_added_elements ++ ++ @property ++ def num_elements_padded(self) -> int: ++ return self.num_org_elements_padded + self.num_added_elements_padded ++ ++ def __post_init__(self): ++ # sanity checks ++ assert self.padded_org_vocab_start_index <= self.padded_org_vocab_end_index ++ assert self.padded_added_vocab_start_index <= self.padded_added_vocab_end_index ++ ++ assert self.org_vocab_start_index <= self.org_vocab_end_index ++ assert self.added_vocab_start_index <= self.added_vocab_end_index ++ ++ assert self.org_vocab_start_index <= self.padded_org_vocab_start_index ++ assert self.added_vocab_start_index <= self.padded_added_vocab_start_index ++ assert self.org_vocab_end_index <= self.padded_org_vocab_end_index ++ assert self.added_vocab_end_index <= self.padded_added_vocab_end_index ++ ++ assert self.num_org_elements <= self.num_org_elements_padded ++ assert self.num_added_elements <= self.num_added_elements_padded ++ ++ ++@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend) ++def get_masked_input_and_mask( ++ input_: torch.Tensor, ++ org_vocab_start_index: int, ++ org_vocab_end_index: int, ++ num_org_vocab_padding: int, ++ added_vocab_start_index: int, ++ added_vocab_end_index: int, ++) -> tuple[torch.Tensor, torch.Tensor]: ++ # torch.compile will fuse all of the pointwise ops below ++ # into a single kernel, making it very fast ++ org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ < org_vocab_end_index) ++ added_vocab_mask = (input_ >= added_vocab_start_index) & ( ++ input_ < added_vocab_end_index ++ ) ++ added_offset = ( ++ added_vocab_start_index ++ - (org_vocab_end_index - org_vocab_start_index) ++ - num_org_vocab_padding ++ ) ++ valid_offset = (org_vocab_start_index * org_vocab_mask) + ( ++ added_offset * added_vocab_mask ++ ) ++ vocab_mask = org_vocab_mask | added_vocab_mask ++ input_ = vocab_mask * (input_ - valid_offset) ++ return input_, ~vocab_mask ++ ++ ++# --8<-- [start:vocab_parallel_embedding] ++@PluggableLayer.register("vocab_parallel_embedding") ++class VocabParallelEmbedding(PluggableLayer): ++ """Embedding parallelized in the vocabulary dimension. ++ ++ Adapted from torch.nn.Embedding, note that we pad the vocabulary size to ++ make sure it is divisible by the number of model parallel GPUs. ++ ++ In order to support various loading methods, we ensure that LoRA-added ++ embeddings are always at the end of TP-sharded tensors. In other words, ++ we shard base embeddings and LoRA embeddings separately (both padded), ++ and place them in the same tensor. ++ In this example, we will have the original vocab size = 1010, ++ added vocab size = 16 and padding to 64. Therefore, the total ++ vocab size with padding will be 1088 (because we first pad 1010 to ++ 1024, add 16, and then pad to 1088). ++ Therefore, the tensor format looks like the following: ++ TP1, rank 0 (no sharding): ++ |< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >| ++ corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1025 | -1 | ... | -1 | ++ index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 | ++ ++ TP2, rank 0: ++ |< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >| ++ corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1010 | ... | 1025 | -1 | ... | -1 | ++ index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 | ++ TP2, rank 1: ++ |< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >| ++ corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 | ++ index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 | ++ ++ Args: ++ num_embeddings: vocabulary size. ++ embedding_dim: size of hidden state. ++ params_dtype: type of the parameters. ++ org_num_embeddings: original vocabulary size (without LoRA). ++ padding_size: padding size for the vocabulary. ++ quant_config: quant config for the layer ++ prefix: full name of the layer in the state dict ++ disable_tp: If true, tensor parallelism will be disabled for this layer. ++ """ # noqa: E501 ++ ++ # --8<-- [end:vocab_parallel_embedding] ++ ++ def __init__( ++ self, ++ num_embeddings: int, ++ embedding_dim: int, ++ params_dtype: torch.dtype | None = None, ++ org_num_embeddings: int | None = None, ++ padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, ++ quant_config: QuantizationConfig | None = None, ++ prefix: str = "", + *, + disable_tp: bool = False, +- ): +- super().__init__() +- +- # Keep the input dimensions. +- self.disable_tp = disable_tp +- if disable_tp: +- tp_rank, self.tp_size = 0, 1 +- else: +- tp_rank = get_tensor_model_parallel_rank() +- self.tp_size = get_tensor_model_parallel_world_size() +- self.tp_rank = tp_rank +- self.num_embeddings = num_embeddings +- self.padding_size = padding_size +- self.org_vocab_size = org_num_embeddings or num_embeddings +- num_added_embeddings = num_embeddings - self.org_vocab_size +- self.org_vocab_size_padded = pad_vocab_size( +- self.org_vocab_size, self.padding_size +- ) +- self.num_embeddings_padded = pad_vocab_size( +- self.org_vocab_size_padded + num_added_embeddings, self.padding_size +- ) +- assert self.org_vocab_size_padded <= self.num_embeddings_padded +- +- self.shard_indices = self._get_indices( +- self.num_embeddings_padded, +- self.org_vocab_size_padded, +- self.num_embeddings, +- self.org_vocab_size, +- tp_rank, +- self.tp_size, +- ) +- self.embedding_dim = embedding_dim +- +- quant_method = None +- if quant_config is not None: +- quant_method = quant_config.get_quant_method(self, prefix=prefix) +- if quant_method is None: +- quant_method = UnquantizedEmbeddingMethod() +- +- # If we are making an embedding layer, then our quantization linear +- # method must implement the embedding operation. If we are another ++ lm_head_quantization: Literal["nvfp4"] | None = None, ++ ): ++ super().__init__() ++ ++ # Keep the input dimensions. ++ self.disable_tp = disable_tp ++ if disable_tp: ++ tp_rank, self.tp_size = 0, 1 ++ else: ++ tp_rank = get_tensor_model_parallel_rank() ++ self.tp_size = get_tensor_model_parallel_world_size() ++ self.tp_rank = tp_rank ++ self.num_embeddings = num_embeddings ++ self.padding_size = padding_size ++ self.org_vocab_size = org_num_embeddings or num_embeddings ++ num_added_embeddings = num_embeddings - self.org_vocab_size ++ self.org_vocab_size_padded = pad_vocab_size( ++ self.org_vocab_size, self.padding_size ++ ) ++ self.num_embeddings_padded = pad_vocab_size( ++ self.org_vocab_size_padded + num_added_embeddings, self.padding_size ++ ) ++ assert self.org_vocab_size_padded <= self.num_embeddings_padded ++ ++ self.shard_indices = self._get_indices( ++ self.num_embeddings_padded, ++ self.org_vocab_size_padded, ++ self.num_embeddings, ++ self.org_vocab_size, ++ tp_rank, ++ self.tp_size, ++ ) ++ self.embedding_dim = embedding_dim ++ ++ quant_method = None ++ if quant_config is not None: ++ quant_method = quant_config.get_quant_method(self, prefix=prefix) ++ if quant_method is None: ++ quant_method = UnquantizedEmbeddingMethod() ++ ++ # If we are making an embedding layer, then our quantization linear ++ # method must implement the embedding operation. If we are another + # layer type like ParallelLMHead, this is not important. + is_embedding_layer = not isinstance(self, ParallelLMHead) +- quant_method_implements_embedding = method_has_implemented_embedding( +- type(quant_method) +- ) +- if is_embedding_layer and not quant_method_implements_embedding: +- raise NotImplementedError( +- f"The class {type(quant_method).__name__} must implement " +- "the 'embedding' method, see UnquantizedEmbeddingMethod." ++ self.runtime_lm_head_quantization: Literal["nvfp4"] | None = None ++ if not is_embedding_layer and lm_head_quantization is not None: ++ from vllm.model_executor.layers.linear import UnquantizedLinearMethod ++ from vllm.model_executor.layers.quantization.online.nvfp4 import ( ++ Nvfp4OnlineLinearMethod, + ) + +- self.quant_method: QuantizeMethodBase = quant_method +- +- if params_dtype is None: +- params_dtype = torch.get_default_dtype() +- self.params_dtype = params_dtype +- # Divide the weight matrix along the vocabulary dimension. +- self.num_added_embeddings = self.num_embeddings - self.org_vocab_size +- self.num_embeddings_per_partition = divide( +- self.num_embeddings_padded, self.tp_size +- ) +- assert ( +- self.shard_indices.num_elements_padded == self.num_embeddings_per_partition +- ) +- self.num_org_embeddings_per_partition = ( +- self.shard_indices.org_vocab_end_index +- - self.shard_indices.org_vocab_start_index +- ) +- self.num_added_embeddings_per_partition = ( +- self.shard_indices.added_vocab_end_index +- - self.shard_indices.added_vocab_start_index +- ) +- +- self.quant_method.create_weights( +- self, +- self.embedding_dim, +- [self.num_embeddings_per_partition], +- self.embedding_dim, +- self.num_embeddings_padded, +- params_dtype=params_dtype, +- weight_loader=self.weight_loader, +- ) +- self.update_param_tp_status() +- +- def update_param_tp_status(self): +- for param in self.parameters(): +- if isinstance(param, BasevLLMParameter): +- param.tp_rank = self.tp_rank +- param.tp_size = self.tp_size +- +- @classmethod +- def _get_indices( +- cls, +- vocab_size_padded: int, +- org_vocab_size_padded: int, +- vocab_size: int, +- org_vocab_size: int, +- tp_rank: int, +- tp_size: int, +- ) -> VocabParallelEmbeddingShardIndices: +- """Get start and end indices for vocab parallel embedding, following the +- layout outlined in the class docstring, based on the given tp_rank and +- tp_size.""" +- num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded +- padded_org_vocab_start_index, padded_org_vocab_end_index = ( +- vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, tp_size) +- ) +- padded_added_vocab_start_index, padded_added_vocab_end_index = ( +- vocab_range_from_global_vocab_size( +- num_added_embeddings_padded, tp_rank, tp_size, offset=org_vocab_size +- ) +- ) +- # remove padding +- org_vocab_start_index = min(padded_org_vocab_start_index, org_vocab_size) +- org_vocab_end_index = min(padded_org_vocab_end_index, org_vocab_size) +- added_vocab_start_index = min(padded_added_vocab_start_index, vocab_size) +- added_vocab_end_index = min(padded_added_vocab_end_index, vocab_size) +- return VocabParallelEmbeddingShardIndices( +- padded_org_vocab_start_index, +- padded_org_vocab_end_index, +- padded_added_vocab_start_index, +- padded_added_vocab_end_index, +- org_vocab_start_index, +- org_vocab_end_index, +- added_vocab_start_index, +- added_vocab_end_index, +- ) +- +- def get_sharded_to_full_mapping(self) -> list[int] | None: +- """Get a mapping that can be used to reindex the gathered +- logits for sampling. +- +- During sampling, we gather logits from all ranks. The relationship +- of index->token_id will follow the same format as outlined in the class +- docstring. However, after the gather, we want to reindex the final +- logits tensor to map index->token_id one-to-one (the index is always +- equal the token_id it corresponds to). The indices returned by this +- method allow us to do that. +- """ +- if self.tp_size < 2: +- return None +- +- base_embeddings: list[int] = [] +- added_embeddings: list[int] = [] +- padding: list[int] = [] +- for tp_rank in range(self.tp_size): +- shard_indices = self._get_indices( +- self.num_embeddings_padded, +- self.org_vocab_size_padded, +- self.num_embeddings, +- self.org_vocab_size, +- tp_rank, +- self.tp_size, +- ) +- range_start = self.num_embeddings_per_partition * tp_rank +- range_end = self.num_embeddings_per_partition * (tp_rank + 1) +- base_embeddings.extend( +- range(range_start, range_start + shard_indices.num_org_elements) +- ) +- padding.extend( +- range( +- range_start + shard_indices.num_org_elements, +- range_start + shard_indices.num_org_elements_padded, +- ) +- ) +- added_embeddings.extend( +- range( +- range_start + shard_indices.num_org_elements_padded, +- range_start +- + shard_indices.num_org_elements_padded +- + shard_indices.num_added_elements, +- ) +- ) +- padding.extend( +- range( +- range_start +- + shard_indices.num_org_elements_padded +- + shard_indices.num_added_elements, +- range_start +- + shard_indices.num_org_elements_padded +- + shard_indices.num_added_elements_padded, +- ) +- ) +- assert ( +- range_start +- + shard_indices.num_org_elements_padded +- + shard_indices.num_added_elements_padded +- == range_end +- ) +- ret = base_embeddings + added_embeddings + padding +- assert len(ret) == self.num_embeddings_padded +- return ret +- +- def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): +- output_dim = getattr(param, "output_dim", None) +- packed_dim = getattr(param, "packed_dim", None) +- +- # If parameter does not have output dim, then it should +- # be copied onto all gpus (e.g. g_idx for act_order gptq). +- if output_dim is None: +- if ( +- loaded_weight.ndim == 0 +- and param.data.ndim == 1 +- and param.data.numel() == 1 ++ if not isinstance( ++ quant_method, ++ (UnquantizedEmbeddingMethod, UnquantizedLinearMethod), + ): +- loaded_weight = loaded_weight.reshape(1) +- assert param.data.shape == loaded_weight.shape +- param.data.copy_(loaded_weight) +- return +- +- # Shard indexes for loading the weight +- start_idx = self.shard_indices.org_vocab_start_index +- shard_size = self.shard_indices.org_vocab_end_index - start_idx +- +- # If param packed on the same dim we are sharding on, then +- # need to adjust offsets of loaded weight by pack_factor. +- if packed_dim is not None and packed_dim == output_dim: +- packed_factor = ( +- param.packed_factor +- if isinstance(param, BasevLLMParameter) +- else param.pack_factor +- ) +- assert loaded_weight.shape[output_dim] == ( +- self.org_vocab_size // param.packed_factor +- ) +- start_idx = start_idx // packed_factor +- shard_size = shard_size // packed_factor +- else: +- assert loaded_weight.shape[output_dim] == self.org_vocab_size +- +- # Copy the data. Select chunk corresponding to current shard. +- loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) +- param[: loaded_weight.shape[0]].data.copy_(loaded_weight) +- param[loaded_weight.shape[0] :].data.fill_(0) +- +- def forward(self, input_): +- if self.tp_size > 1: +- # Build the mask. +- masked_input, input_mask = get_masked_input_and_mask( +- input_, +- self.shard_indices.org_vocab_start_index, +- self.shard_indices.org_vocab_end_index, +- self.shard_indices.num_org_vocab_padding, +- self.shard_indices.added_vocab_start_index, +- self.shard_indices.added_vocab_end_index, ++ raise ValueError( ++ "Runtime NVFP4 proposal-head quantization requires an " ++ "unquantized checkpoint head" ++ ) ++ quant_method = Nvfp4OnlineLinearMethod( ++ use_a16=envs.VLLM_LM_HEAD_A16 + ) +- else: +- masked_input = input_ +- # Get the embeddings. +- output_parallel = self.quant_method.embedding(self, masked_input.long()) +- # Mask the output embedding. +- if self.tp_size > 1: +- output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0) +- # Reduce across all the model parallel GPUs. +- return tensor_model_parallel_all_reduce(output_parallel) +- return output_parallel +- +- def extra_repr(self) -> str: +- s = f"num_embeddings={self.num_embeddings}" +- s += f", num_embeddings_per_partition={self.num_embeddings_per_partition}" +- s += f", embedding_dim={self.embedding_dim}" +- s += f", org_vocab_size={self.org_vocab_size}" +- s += f", num_embeddings_padded={self.num_embeddings_padded}" +- s += f", tp_size={self.tp_size}" +- return s +- +- +-# --8<-- [start:parallel_lm_head] +-@PluggableLayer.register("parallel_lm_head") +-class ParallelLMHead(VocabParallelEmbedding): +- """Parallelized LM head. +- +- Output logits weight matrices used in the Sampler. The weight and bias +- tensors are padded to make sure they are divisible by the number of +- model parallel GPUs. +- +- Args: +- num_embeddings: vocabulary size. +- embedding_dim: size of hidden state. +- bias: whether to use bias. +- params_dtype: type of the parameters. +- org_num_embeddings: original vocabulary size (without LoRA). +- padding_size: padding size for the vocabulary. +- disable_tp: If true, tensor parallelism will be disabled for this layer. +- """ +- +- # --8<-- [end:parallel_lm_head] +- +- def __init__( +- self, +- num_embeddings: int, +- embedding_dim: int, +- bias: bool = False, +- params_dtype: torch.dtype | None = None, +- org_num_embeddings: int | None = None, +- padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, +- quant_config: QuantizationConfig | None = None, +- prefix: str = "", ++ self.runtime_lm_head_quantization = "nvfp4" ++ quant_method_implements_embedding = method_has_implemented_embedding( ++ type(quant_method) ++ ) ++ if is_embedding_layer and not quant_method_implements_embedding: ++ raise NotImplementedError( ++ f"The class {type(quant_method).__name__} must implement " ++ "the 'embedding' method, see UnquantizedEmbeddingMethod." ++ ) ++ ++ self.quant_method: QuantizeMethodBase = quant_method ++ ++ if params_dtype is None: ++ params_dtype = torch.get_default_dtype() ++ self.params_dtype = params_dtype ++ # Divide the weight matrix along the vocabulary dimension. ++ self.num_added_embeddings = self.num_embeddings - self.org_vocab_size ++ self.num_embeddings_per_partition = divide( ++ self.num_embeddings_padded, self.tp_size ++ ) ++ assert ( ++ self.shard_indices.num_elements_padded == self.num_embeddings_per_partition ++ ) ++ self.num_org_embeddings_per_partition = ( ++ self.shard_indices.org_vocab_end_index ++ - self.shard_indices.org_vocab_start_index ++ ) ++ self.num_added_embeddings_per_partition = ( ++ self.shard_indices.added_vocab_end_index ++ - self.shard_indices.added_vocab_start_index ++ ) ++ ++ self.quant_method.create_weights( ++ self, ++ self.embedding_dim, ++ [self.num_embeddings_per_partition], ++ self.embedding_dim, ++ self.num_embeddings_padded, ++ params_dtype=params_dtype, ++ weight_loader=self.weight_loader, ++ ) ++ self.update_param_tp_status() ++ ++ def update_param_tp_status(self): ++ for param in self.parameters(): ++ if isinstance(param, BasevLLMParameter): ++ param.tp_rank = self.tp_rank ++ param.tp_size = self.tp_size ++ ++ @classmethod ++ def _get_indices( ++ cls, ++ vocab_size_padded: int, ++ org_vocab_size_padded: int, ++ vocab_size: int, ++ org_vocab_size: int, ++ tp_rank: int, ++ tp_size: int, ++ ) -> VocabParallelEmbeddingShardIndices: ++ """Get start and end indices for vocab parallel embedding, following the ++ layout outlined in the class docstring, based on the given tp_rank and ++ tp_size.""" ++ num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded ++ padded_org_vocab_start_index, padded_org_vocab_end_index = ( ++ vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, tp_size) ++ ) ++ padded_added_vocab_start_index, padded_added_vocab_end_index = ( ++ vocab_range_from_global_vocab_size( ++ num_added_embeddings_padded, tp_rank, tp_size, offset=org_vocab_size ++ ) ++ ) ++ # remove padding ++ org_vocab_start_index = min(padded_org_vocab_start_index, org_vocab_size) ++ org_vocab_end_index = min(padded_org_vocab_end_index, org_vocab_size) ++ added_vocab_start_index = min(padded_added_vocab_start_index, vocab_size) ++ added_vocab_end_index = min(padded_added_vocab_end_index, vocab_size) ++ return VocabParallelEmbeddingShardIndices( ++ padded_org_vocab_start_index, ++ padded_org_vocab_end_index, ++ padded_added_vocab_start_index, ++ padded_added_vocab_end_index, ++ org_vocab_start_index, ++ org_vocab_end_index, ++ added_vocab_start_index, ++ added_vocab_end_index, ++ ) ++ ++ def get_sharded_to_full_mapping(self) -> list[int] | None: ++ """Get a mapping that can be used to reindex the gathered ++ logits for sampling. ++ ++ During sampling, we gather logits from all ranks. The relationship ++ of index->token_id will follow the same format as outlined in the class ++ docstring. However, after the gather, we want to reindex the final ++ logits tensor to map index->token_id one-to-one (the index is always ++ equal the token_id it corresponds to). The indices returned by this ++ method allow us to do that. ++ """ ++ if self.tp_size < 2: ++ return None ++ ++ base_embeddings: list[int] = [] ++ added_embeddings: list[int] = [] ++ padding: list[int] = [] ++ for tp_rank in range(self.tp_size): ++ shard_indices = self._get_indices( ++ self.num_embeddings_padded, ++ self.org_vocab_size_padded, ++ self.num_embeddings, ++ self.org_vocab_size, ++ tp_rank, ++ self.tp_size, ++ ) ++ range_start = self.num_embeddings_per_partition * tp_rank ++ range_end = self.num_embeddings_per_partition * (tp_rank + 1) ++ base_embeddings.extend( ++ range(range_start, range_start + shard_indices.num_org_elements) ++ ) ++ padding.extend( ++ range( ++ range_start + shard_indices.num_org_elements, ++ range_start + shard_indices.num_org_elements_padded, ++ ) ++ ) ++ added_embeddings.extend( ++ range( ++ range_start + shard_indices.num_org_elements_padded, ++ range_start ++ + shard_indices.num_org_elements_padded ++ + shard_indices.num_added_elements, ++ ) ++ ) ++ padding.extend( ++ range( ++ range_start ++ + shard_indices.num_org_elements_padded ++ + shard_indices.num_added_elements, ++ range_start ++ + shard_indices.num_org_elements_padded ++ + shard_indices.num_added_elements_padded, ++ ) ++ ) ++ assert ( ++ range_start ++ + shard_indices.num_org_elements_padded ++ + shard_indices.num_added_elements_padded ++ == range_end ++ ) ++ ret = base_embeddings + added_embeddings + padding ++ assert len(ret) == self.num_embeddings_padded ++ return ret ++ ++ def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): ++ output_dim = getattr(param, "output_dim", None) ++ packed_dim = getattr(param, "packed_dim", None) ++ ++ # If parameter does not have output dim, then it should ++ # be copied onto all gpus (e.g. g_idx for act_order gptq). ++ if output_dim is None: ++ if ( ++ loaded_weight.ndim == 0 ++ and param.data.ndim == 1 ++ and param.data.numel() == 1 ++ ): ++ loaded_weight = loaded_weight.reshape(1) ++ assert param.data.shape == loaded_weight.shape ++ param.data.copy_(loaded_weight) ++ return ++ ++ # Shard indexes for loading the weight ++ start_idx = self.shard_indices.org_vocab_start_index ++ shard_size = self.shard_indices.org_vocab_end_index - start_idx ++ ++ # If param packed on the same dim we are sharding on, then ++ # need to adjust offsets of loaded weight by pack_factor. ++ if packed_dim is not None and packed_dim == output_dim: ++ packed_factor = ( ++ param.packed_factor ++ if isinstance(param, BasevLLMParameter) ++ else param.pack_factor ++ ) ++ assert loaded_weight.shape[output_dim] == ( ++ self.org_vocab_size // param.packed_factor ++ ) ++ start_idx = start_idx // packed_factor ++ shard_size = shard_size // packed_factor ++ else: ++ assert loaded_weight.shape[output_dim] == self.org_vocab_size ++ ++ # Copy the data. Select chunk corresponding to current shard. ++ loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) ++ param[: loaded_weight.shape[0]].data.copy_(loaded_weight) ++ param[loaded_weight.shape[0] :].data.fill_(0) ++ ++ def forward(self, input_): ++ if self.tp_size > 1: ++ # Build the mask. ++ masked_input, input_mask = get_masked_input_and_mask( ++ input_, ++ self.shard_indices.org_vocab_start_index, ++ self.shard_indices.org_vocab_end_index, ++ self.shard_indices.num_org_vocab_padding, ++ self.shard_indices.added_vocab_start_index, ++ self.shard_indices.added_vocab_end_index, ++ ) ++ else: ++ masked_input = input_ ++ # Get the embeddings. ++ output_parallel = self.quant_method.embedding(self, masked_input.long()) ++ # Mask the output embedding. ++ if self.tp_size > 1: ++ output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0) ++ # Reduce across all the model parallel GPUs. ++ return tensor_model_parallel_all_reduce(output_parallel) ++ return output_parallel ++ ++ def extra_repr(self) -> str: ++ s = f"num_embeddings={self.num_embeddings}" ++ s += f", num_embeddings_per_partition={self.num_embeddings_per_partition}" ++ s += f", embedding_dim={self.embedding_dim}" ++ s += f", org_vocab_size={self.org_vocab_size}" ++ s += f", num_embeddings_padded={self.num_embeddings_padded}" ++ s += f", tp_size={self.tp_size}" ++ return s ++ ++ ++# --8<-- [start:parallel_lm_head] ++@PluggableLayer.register("parallel_lm_head") ++class ParallelLMHead(VocabParallelEmbedding): ++ """Parallelized LM head. ++ ++ Output logits weight matrices used in the Sampler. The weight and bias ++ tensors are padded to make sure they are divisible by the number of ++ model parallel GPUs. ++ ++ Args: ++ num_embeddings: vocabulary size. ++ embedding_dim: size of hidden state. ++ bias: whether to use bias. ++ params_dtype: type of the parameters. ++ org_num_embeddings: original vocabulary size (without LoRA). ++ padding_size: padding size for the vocabulary. ++ disable_tp: If true, tensor parallelism will be disabled for this layer. ++ """ ++ ++ # --8<-- [end:parallel_lm_head] ++ ++ def __init__( ++ self, ++ num_embeddings: int, ++ embedding_dim: int, ++ bias: bool = False, ++ params_dtype: torch.dtype | None = None, ++ org_num_embeddings: int | None = None, ++ padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, ++ quant_config: QuantizationConfig | None = None, ++ prefix: str = "", + *, + disable_tp: bool = False, +- ): +- super().__init__( +- num_embeddings, +- embedding_dim, +- params_dtype, +- org_num_embeddings, +- padding_size, +- quant_config, ++ lm_head_quantization: Literal["nvfp4"] | None = None, ++ ): ++ super().__init__( ++ num_embeddings, ++ embedding_dim, ++ params_dtype, ++ org_num_embeddings, ++ padding_size, ++ quant_config, + prefix, + disable_tp=disable_tp, ++ lm_head_quantization=lm_head_quantization, + ) +- self.quant_config = quant_config +- if bias: +- self._register_bias() +- else: +- self.register_parameter("bias", None) +- +- def _register_bias(self): +- data = torch.empty(self.num_embeddings_per_partition, dtype=self.params_dtype) +- self.bias = Parameter(data, requires_grad=False) +- weight_attrs = dict(output_dim=0, weight_loader=self.weight_loader) +- set_weight_attrs(weight=self.bias, weight_attrs=weight_attrs) +- +- def tie_weights(self, embed_tokens: VocabParallelEmbedding): +- """Tie the weights with word embeddings.""" +- return self.quant_method.tie_weights(self, embed_tokens) +- +- def forward(self, input_): +- del input_ +- raise RuntimeError("LMHead's weights should be used in the sampler.") ++ self.quant_config = quant_config ++ if bias: ++ self._register_bias() ++ else: ++ self.register_parameter("bias", None) ++ ++ def _register_bias(self): ++ data = torch.empty(self.num_embeddings_per_partition, dtype=self.params_dtype) ++ self.bias = Parameter(data, requires_grad=False) ++ weight_attrs = dict(output_dim=0, weight_loader=self.weight_loader) ++ set_weight_attrs(weight=self.bias, weight_attrs=weight_attrs) ++ ++ def tie_weights(self, embed_tokens: VocabParallelEmbedding): ++ """Tie the weights with word embeddings.""" ++ return self.quant_method.tie_weights(self, embed_tokens) ++ ++ def forward(self, input_): ++ del input_ ++ raise RuntimeError("LMHead's weights should be used in the sampler.") +diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py +index 8ac43e0bb2a17957ac3f013c4da75d030fae0502..e200fd64483419ee72adc107be580519fbfb38f8 100644 +--- a/vllm/model_executor/models/deepseek_mtp.py ++++ b/vllm/model_executor/models/deepseek_mtp.py +@@ -1,547 +1,550 @@ +-# SPDX-License-Identifier: Apache-2.0 +-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +-import typing +-from collections.abc import Callable, Iterable +- +-import torch +-import torch.nn as nn +-from transformers import PretrainedConfig +- +-from vllm.compilation.decorators import support_torch_compile +-from vllm.config import VllmConfig +-from vllm.distributed import tensor_model_parallel_all_gather +-from vllm.model_executor.layers.fused_moe import ( +- fused_moe_make_expert_params_mapping, +-) +-from vllm.model_executor.layers.fused_moe.utils import ( +- is_model_fused_shared_expert_compatible, +-) +-from vllm.model_executor.layers.layernorm import RMSNorm +-from vllm.model_executor.layers.logits_processor import LogitsProcessor +-from vllm.model_executor.layers.quantization import QuantizationConfig +-from vllm.model_executor.layers.vocab_parallel_embedding import ( +- ParallelLMHead, +- VocabParallelEmbedding, +-) +-from vllm.model_executor.model_loader.mtp_validation import ( +- is_mtp_completeness_check_enabled, +-) +-from vllm.model_executor.model_loader.weight_utils import ( +- default_weight_loader, +- maybe_remap_kv_scale_name, +-) +-from vllm.platforms import current_platform +-from vllm.sequence import IntermediateTensors +- +-from .deepseek_v2 import ( +- DeepseekV2DecoderLayer, +- DeepseekV2MixtureOfExperts, +- DeepseekV2MoE, +- _try_load_fp8_indexer_wk, +-) +-from .utils import ( +- get_pp_missing_layer_names, +- get_spec_layer_idx_from_weight_name, +- maybe_prefix, +-) +- +- +-class SharedHead(nn.Module): ++# SPDX-License-Identifier: Apache-2.0 ++# SPDX-FileCopyrightText: Copyright contributors to the vLLM project ++import typing ++from collections.abc import Callable, Iterable ++ ++import torch ++import torch.nn as nn ++from transformers import PretrainedConfig ++ ++from vllm.compilation.decorators import support_torch_compile ++from vllm.config import VllmConfig ++from vllm.distributed import tensor_model_parallel_all_gather ++from vllm.model_executor.layers.fused_moe import ( ++ fused_moe_make_expert_params_mapping, ++) ++from vllm.model_executor.layers.fused_moe.utils import ( ++ is_model_fused_shared_expert_compatible, ++) ++from vllm.model_executor.layers.layernorm import RMSNorm ++from vllm.model_executor.layers.logits_processor import LogitsProcessor ++from vllm.model_executor.layers.quantization import QuantizationConfig ++from vllm.model_executor.layers.vocab_parallel_embedding import ( ++ ParallelLMHead, ++ VocabParallelEmbedding, ++) ++from vllm.model_executor.model_loader.mtp_validation import ( ++ is_mtp_completeness_check_enabled, ++) ++from vllm.model_executor.model_loader.weight_utils import ( ++ default_weight_loader, ++ maybe_remap_kv_scale_name, ++) ++from vllm.platforms import current_platform ++from vllm.sequence import IntermediateTensors ++ ++from .deepseek_v2 import ( ++ DeepseekV2DecoderLayer, ++ DeepseekV2MixtureOfExperts, ++ DeepseekV2MoE, ++ _try_load_fp8_indexer_wk, ++) ++from .utils import ( ++ get_pp_missing_layer_names, ++ get_spec_layer_idx_from_weight_name, ++ maybe_prefix, ++) ++ ++ ++class SharedHead(nn.Module): + def __init__( + self, + config: PretrainedConfig, + prefix: str, + quant_config: QuantizationConfig | None = None, ++ *, ++ lm_head_quantization: typing.Literal["nvfp4"] | None = None, + ) -> None: +- super().__init__() +- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) +- self.head = ParallelLMHead( +- config.vocab_size, ++ super().__init__() ++ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) ++ self.head = ParallelLMHead( ++ config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "head"), ++ lm_head_quantization=lm_head_quantization, + ) +- +- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: +- return self.norm(hidden_states) +- +- +-class DeepSeekMultiTokenPredictorLayer(nn.Module): +- def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: +- super().__init__() +- +- assert vllm_config.speculative_config is not None +- config = vllm_config.speculative_config.draft_model_config.hf_config +- self.config = config +- quant_config = vllm_config.quant_config +- +- self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) +- self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) +- self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) +- +- self.device = current_platform.device_type +- +- self.is_v32 = hasattr(config, "index_topk") +- if self.is_v32: +- topk_tokens = config.index_topk +- topk_indices_buffer = torch.empty( +- vllm_config.scheduler_config.max_num_batched_tokens, +- topk_tokens, +- dtype=torch.int32, +- device=self.device, +- ) +- else: +- topk_indices_buffer = None +- +- self.shared_head = SharedHead( +- config=config, prefix=prefix, quant_config=quant_config +- ) +- self.mtp_block = DeepseekV2DecoderLayer( +- vllm_config, +- prefix, +- config=self.config, +- topk_indices_buffer=topk_indices_buffer, +- ) +- +- def forward( +- self, +- input_ids: torch.Tensor, +- positions: torch.Tensor, +- previous_hidden_states: torch.Tensor, +- inputs_embeds: torch.Tensor | None = None, +- spec_step_index: int = 0, +- ) -> torch.Tensor: +- assert inputs_embeds is not None +- # masking inputs at position 0, as not needed by MTP +- inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds) +- inputs_embeds = self.enorm(inputs_embeds) +- previous_hidden_states = self.hnorm(previous_hidden_states) +- +- hidden_states = self.eh_proj( +- torch.cat([inputs_embeds, previous_hidden_states], dim=-1) +- ) +- +- hidden_states, residual = self.mtp_block( +- positions=positions, +- hidden_states=hidden_states, +- residual=None, +- ) +- hidden_states = residual + hidden_states # pre-final-norm (logits hidden) +- if self.mtp_block.use_sequence_parallel_moe: +- hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) +- hidden_states = hidden_states[: positions.shape[0]] +- # Recycle the post-final-norm hidden into the next draft step. +- # compute_logits applies shared_head (== final norm) to the pre-norm +- # element, so logits and the recycle each get exactly one final-norm. +- # Matches SGLang's deepseek_nextn. +- return hidden_states, self.shared_head(hidden_states) +- +- +-class DeepSeekMultiTokenPredictor(nn.Module): +- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): +- super().__init__() +- config = vllm_config.model_config.hf_config +- self.mtp_start_layer_idx = config.num_hidden_layers +- self.num_mtp_layers = config.num_nextn_predict_layers +- # to map the exact layer index from weights +- +- self.layers = torch.nn.ModuleDict( +- { +- str(idx): DeepSeekMultiTokenPredictorLayer( +- vllm_config, f"{prefix}.layers.{idx}" +- ) +- for idx in range( +- self.mtp_start_layer_idx, +- self.mtp_start_layer_idx + self.num_mtp_layers, +- ) +- } +- ) +- self.embed_tokens = VocabParallelEmbedding( +- config.vocab_size, +- config.hidden_size, +- prefix=maybe_prefix(prefix, "embed_tokens"), +- ) +- self.logits_processor = LogitsProcessor(config.vocab_size) +- +- def set_skip_topk(self, skip: bool): +- """Toggle skip_topk on all MTP layers with sparse attention. +- +- Called by the proposer to implement index_share_for_mtp_iteration: +- step 0 sets skip=False (compute own indices), steps 1+ set skip=True +- (reuse step 0's indices). +- """ +- for layer in self.layers.values(): +- mtp_block = getattr(layer, "mtp_block", None) +- if mtp_block is not None: +- self_attn = getattr(mtp_block, "self_attn", None) +- if self_attn is not None: +- mla_attn = getattr(self_attn, "mla_attn", None) +- if mla_attn is not None and hasattr(mla_attn, "skip_topk"): +- mla_attn.skip_topk = skip +- +- def compact_topk_indices(self, slot_ids: torch.Tensor): +- """Gather the top-k index rows at ``slot_ids`` to the front of the buffer.""" +- num_slots = slot_ids.numel() +- for layer in self.layers.values(): +- mtp_block = getattr(layer, "mtp_block", None) +- if mtp_block is not None: +- self_attn = getattr(mtp_block, "self_attn", None) +- if self_attn is not None: +- mla_attn = getattr(self_attn, "mla_attn", None) +- if mla_attn is not None and hasattr( +- mla_attn, "topk_indices_buffer" +- ): +- topk_indices_buffer = mla_attn.topk_indices_buffer +- topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] +- +- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: +- return self.embed_tokens(input_ids) +- +- def forward( +- self, +- input_ids: torch.Tensor, +- positions: torch.Tensor, +- previous_hidden_states: torch.Tensor, +- inputs_embeds: torch.Tensor | None = None, +- spec_step_idx: int = 0, +- ) -> torch.Tensor: +- if inputs_embeds is None: +- inputs_embeds = self.embed_tokens(input_ids) +- current_step_idx = spec_step_idx % self.num_mtp_layers +- return self.layers[str(self.mtp_start_layer_idx + current_step_idx)]( +- input_ids, +- positions, +- previous_hidden_states, +- inputs_embeds, +- current_step_idx, +- ) +- +- def compute_logits( +- self, +- hidden_states: torch.Tensor, +- spec_step_idx: int = 0, +- ) -> torch.Tensor: +- current_step_idx = spec_step_idx % self.num_mtp_layers +- mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] +- logits = self.logits_processor( +- mtp_layer.shared_head.head, mtp_layer.shared_head(hidden_states) +- ) +- return logits +- +- +-@support_torch_compile +-class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): +- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): +- super().__init__() +- self.config = vllm_config.model_config.hf_config +- self.quant_config = vllm_config.quant_config +- self.model = DeepSeekMultiTokenPredictor( +- vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") +- ) +- # Set MoE hyperparameters +- self.set_moe_parameters() +- +- def set_moe_parameters(self): +- self.num_moe_layers = self.config.num_nextn_predict_layers +- self.num_expert_groups = self.config.n_group +- +- self.moe_layers = [] +- self.moe_mlp_layers = [] +- example_moe = None +- for layer in self.model.layers.values(): +- assert isinstance(layer, DeepSeekMultiTokenPredictorLayer) +- layer = layer.mtp_block +- assert isinstance(layer, DeepseekV2DecoderLayer) +- if isinstance(layer.mlp, DeepseekV2MoE): +- example_moe = layer.mlp +- self.moe_mlp_layers.append(layer.mlp) +- self.moe_layers.append(layer.mlp.experts) +- self.extract_moe_parameters(example_moe) +- self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( +- self.model.layers.values(), +- DeepseekV2MoE, +- "mtp_block.mlp", +- ) +- +- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: +- return self.model.embed_input_ids(input_ids) +- +- def forward( +- self, +- input_ids: torch.Tensor | None, +- positions: torch.Tensor, +- hidden_states: torch.Tensor, +- intermediate_tensors: IntermediateTensors | None = None, +- inputs_embeds: torch.Tensor | None = None, +- spec_step_idx: int = 0, +- ) -> torch.Tensor: +- hidden_states = self.model( +- input_ids, +- positions, +- hidden_states, +- inputs_embeds, +- spec_step_idx, +- ) +- return hidden_states +- +- def compute_logits( +- self, +- hidden_states: torch.Tensor, +- spec_step_idx: int = 0, +- ) -> torch.Tensor | None: +- return self.model.compute_logits(hidden_states, spec_step_idx) +- +- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: +- stacked_params_mapping = [ +- ("gate_up_proj", "gate_proj", 0), +- ("gate_up_proj", "up_proj", 1), +- ("fused_qkv_a_proj", "q_a_proj", 0), +- ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), +- ] +- +- # Fused indexer wk + weights_proj (shard 0 = wk, shard 1 = weights_proj) +- indexer_fused_mapping = [ +- ("wk_weights_proj", "wk", 0), +- ("wk_weights_proj", "weights_proj", 1), +- ] +- stacked_params_mapping.extend(indexer_fused_mapping) +- +- expert_params_mapping = fused_moe_make_expert_params_mapping( +- self, +- ckpt_gate_proj_name="gate_proj", +- ckpt_down_proj_name="down_proj", +- ckpt_up_proj_name="up_proj", +- num_experts=self.config.n_routed_experts +- + ( +- self.config.n_shared_experts +- if self.is_fused_shared_expert_enabled +- else 0 +- ), +- ) +- +- pp_missing_layer_names = get_pp_missing_layer_names(self) +- params_dict = dict(self.named_parameters()) +- loaded_params: set[str] = set() +- _pending_wk_fp8: dict = {} # FP8 indexer wk dequant buffer +- for name, loaded_weight in weights: +- if "rotary_emb.inv_freq" in name: +- continue +- spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) +- if spec_layer is None: +- continue +- is_fusion_moe_shared_experts_layer = ( +- self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) +- ) +- name = self._rewrite_spec_layer_name(spec_layer, name) +- +- if _try_load_fp8_indexer_wk( +- name, +- loaded_weight, +- _pending_wk_fp8, +- params_dict, +- loaded_params, +- pp_missing_layer_names, +- ): +- continue +- +- for param_name, weight_name, shard_id in stacked_params_mapping: +- # Skip non-stacked layers and experts (experts handled below). +- if weight_name not in name: +- continue +- # We have mlp.experts[0].gate_proj in the checkpoint. +- # Since we handle the experts below in expert_params_mapping, +- # we need to skip here BEFORE we update the name, otherwise +- # name will be updated to mlp.experts[0].gate_up_proj, which +- # will then be updated below in expert_params_mapping +- # for mlp.experts[0].gate_gate_up_proj, which breaks load. +- if ("mlp.experts." in name) and name not in params_dict: +- continue +- if is_fusion_moe_shared_experts_layer: +- continue +- name_mapped = name.replace(weight_name, param_name) +- +- # QKV fusion is optional, fall back to normal +- # weight loading if it's not enabled +- if ( +- param_name == "fused_qkv_a_proj" +- ) and name_mapped not in params_dict: +- continue +- else: +- name = name_mapped +- +- # Skip loading extra bias for GPTQ models. +- if name.endswith(".bias") and name not in params_dict: +- continue +- +- param = params_dict[name] +- weight_loader = param.weight_loader +- weight_loader(param, loaded_weight, shard_id) +- break +- else: +- # Special handling: when AITER fusion_shared_experts is enabled, +- # checkpoints may provide a single widened shared_experts tensor +- # without explicit expert indices +- # (e.g. ...mlp.shared_experts.gate_proj.weight). +- # For models with multiple shared experts, split that tensor +- # evenly into per-shared-expert slices and load them into +- # appended expert slots mlp.experts.{n_routed_experts + j}.* +- # accordingly. +- num_chunks = 1 +- if is_fusion_moe_shared_experts_layer: +- num_chunks = getattr(self.config, "n_shared_experts", 1) or 1 +- # Determine split axis based on op type +- # gate/up: ColumnParallel → split along dim 0 +- # down: RowParallel → split along dim 1 +- split_dim = ( +- 1 +- if ("down_proj.weight" in name and loaded_weight.ndim > 1) +- else 0 +- ) +- total = loaded_weight.shape[split_dim] +- assert total % num_chunks == 0, ( +- f"Shared expert weight dim {total} " +- f"not divisible by num_chunks {num_chunks}" +- ) +- chunk_size = total // num_chunks +- +- for j in range(num_chunks): +- chunk_name = name +- weight_to_load = loaded_weight +- +- if is_fusion_moe_shared_experts_layer: +- chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size) +- if loaded_weight.ndim == 1: +- weight_to_load = loaded_weight[chunk_slice] +- elif split_dim == 0: +- weight_to_load = loaded_weight[chunk_slice, :] +- else: +- weight_to_load = loaded_weight[:, chunk_slice] +- # Synthesize an expert-style name so expert mapping +- # can route it +- chunk_name = name.replace( +- "mlp.shared_experts", +- f"mlp.experts.{self.config.n_routed_experts + j}", +- ) +- +- # Use expert_params_mapping to locate the destination +- # param and delegate to its expert-aware weight_loader +- # with expert_id. +- is_expert_weight = False +- for mapping in expert_params_mapping: +- param_name, weight_name, expert_id, expert_shard_id = mapping +- if weight_name not in chunk_name: +- continue +- +- # Anyway, this is an expert weight and should not be +- # attempted to load as other weights later +- is_expert_weight = True +- +- # Do not modify `name` since the loop may continue here +- # Instead, create a new variable +- name_mapped = chunk_name.replace(weight_name, param_name) +- +- param = params_dict[name_mapped] +- # We should ask the weight loader to return success or +- # not here since otherwise we may skip experts with +- # other available replicas. +- weight_loader = typing.cast( +- Callable[..., bool], param.weight_loader +- ) +- success = weight_loader( +- param, +- weight_to_load, +- name_mapped, +- shard_id=expert_shard_id, +- expert_id=expert_id, +- return_success=True, +- ) +- if success: +- if not is_fusion_moe_shared_experts_layer: +- name = name_mapped +- else: +- loaded_params.add(name_mapped) +- break +- else: +- if is_expert_weight: +- # We've checked that this is an expert weight +- # However it's not mapped locally to this rank +- # So we simply skip it +- continue +- +- # Skip loading extra bias for GPTQ models. +- if name.endswith(".bias") and name not in params_dict: +- continue +- +- remapped_name = maybe_remap_kv_scale_name(name, params_dict) +- if remapped_name is None: +- continue +- name = remapped_name +- +- # According to DeepSeek-V3 Technical Report, MTP modules +- # shares embedding layer. We only load the first weights. +- if ( +- spec_layer != self.model.mtp_start_layer_idx +- and ".layers" not in name +- ): +- continue +- +- param = params_dict[name] +- weight_loader = getattr( +- param, "weight_loader", default_weight_loader +- ) +- weight_loader(param, loaded_weight) +- if not is_fusion_moe_shared_experts_layer: +- loaded_params.add(name) +- +- # Validate that weights were loaded for each expected MTP layer. +- loaded_layers: set[int] = set() +- for param_name in loaded_params: +- spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name) +- if spec_layer is not None: +- loaded_layers.add(spec_layer) +- for layer_idx in range( +- self.model.mtp_start_layer_idx, +- self.model.mtp_start_layer_idx + self.model.num_mtp_layers, +- ): +- if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled(): +- raise ValueError( +- f"MTP speculative decoding layer {layer_idx} weights " +- f"missing from checkpoint. The checkpoint may have " +- f"been quantized without including the MTP layers. " +- f"Use a checkpoint that includes MTP layer weights, " +- f"or disable speculative decoding." +- ) +- +- return loaded_params +- +- def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: +- """ +- Rewrite the weight name to match the format of the original model. +- Add .mtp_block for modules in transformer layer block for spec layer +- and rename shared layer weights to be top level. +- """ +- spec_layer_weight_names = [ +- "embed_tokens", +- "enorm", +- "hnorm", +- "eh_proj", +- "shared_head", +- ] +- shared_weight_names = ["embed_tokens"] +- spec_layer_weight = False +- shared_weight = False +- for weight_name in spec_layer_weight_names: +- if weight_name in name: +- spec_layer_weight = True +- if weight_name in shared_weight_names: +- shared_weight = True +- break +- if not spec_layer_weight: +- # treat rest weights as weights for transformer layer block +- name = name.replace( +- f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block." +- ) +- elif shared_weight: +- # treat shared weights as top level weights +- name = name.replace(f"model.layers.{spec_layer}.", "model.") +- return name ++ ++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: ++ return self.norm(hidden_states) ++ ++ ++class DeepSeekMultiTokenPredictorLayer(nn.Module): ++ def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: ++ super().__init__() ++ ++ assert vllm_config.speculative_config is not None ++ config = vllm_config.speculative_config.draft_model_config.hf_config ++ self.config = config ++ quant_config = vllm_config.quant_config ++ ++ self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) ++ self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) ++ self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) ++ ++ self.device = current_platform.device_type ++ ++ self.is_v32 = hasattr(config, "index_topk") ++ if self.is_v32: ++ topk_tokens = config.index_topk ++ topk_indices_buffer = torch.empty( ++ vllm_config.scheduler_config.max_num_batched_tokens, ++ topk_tokens, ++ dtype=torch.int32, ++ device=self.device, ++ ) ++ else: ++ topk_indices_buffer = None ++ ++ self.shared_head = SharedHead( ++ config=config, prefix=prefix, quant_config=quant_config ++ ) ++ self.mtp_block = DeepseekV2DecoderLayer( ++ vllm_config, ++ prefix, ++ config=self.config, ++ topk_indices_buffer=topk_indices_buffer, ++ ) ++ ++ def forward( ++ self, ++ input_ids: torch.Tensor, ++ positions: torch.Tensor, ++ previous_hidden_states: torch.Tensor, ++ inputs_embeds: torch.Tensor | None = None, ++ spec_step_index: int = 0, ++ ) -> torch.Tensor: ++ assert inputs_embeds is not None ++ # masking inputs at position 0, as not needed by MTP ++ inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds) ++ inputs_embeds = self.enorm(inputs_embeds) ++ previous_hidden_states = self.hnorm(previous_hidden_states) ++ ++ hidden_states = self.eh_proj( ++ torch.cat([inputs_embeds, previous_hidden_states], dim=-1) ++ ) ++ ++ hidden_states, residual = self.mtp_block( ++ positions=positions, ++ hidden_states=hidden_states, ++ residual=None, ++ ) ++ hidden_states = residual + hidden_states # pre-final-norm (logits hidden) ++ if self.mtp_block.use_sequence_parallel_moe: ++ hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) ++ hidden_states = hidden_states[: positions.shape[0]] ++ # Recycle the post-final-norm hidden into the next draft step. ++ # compute_logits applies shared_head (== final norm) to the pre-norm ++ # element, so logits and the recycle each get exactly one final-norm. ++ # Matches SGLang's deepseek_nextn. ++ return hidden_states, self.shared_head(hidden_states) ++ ++ ++class DeepSeekMultiTokenPredictor(nn.Module): ++ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): ++ super().__init__() ++ config = vllm_config.model_config.hf_config ++ self.mtp_start_layer_idx = config.num_hidden_layers ++ self.num_mtp_layers = config.num_nextn_predict_layers ++ # to map the exact layer index from weights ++ ++ self.layers = torch.nn.ModuleDict( ++ { ++ str(idx): DeepSeekMultiTokenPredictorLayer( ++ vllm_config, f"{prefix}.layers.{idx}" ++ ) ++ for idx in range( ++ self.mtp_start_layer_idx, ++ self.mtp_start_layer_idx + self.num_mtp_layers, ++ ) ++ } ++ ) ++ self.embed_tokens = VocabParallelEmbedding( ++ config.vocab_size, ++ config.hidden_size, ++ prefix=maybe_prefix(prefix, "embed_tokens"), ++ ) ++ self.logits_processor = LogitsProcessor(config.vocab_size) ++ ++ def set_skip_topk(self, skip: bool): ++ """Toggle skip_topk on all MTP layers with sparse attention. ++ ++ Called by the proposer to implement index_share_for_mtp_iteration: ++ step 0 sets skip=False (compute own indices), steps 1+ set skip=True ++ (reuse step 0's indices). ++ """ ++ for layer in self.layers.values(): ++ mtp_block = getattr(layer, "mtp_block", None) ++ if mtp_block is not None: ++ self_attn = getattr(mtp_block, "self_attn", None) ++ if self_attn is not None: ++ mla_attn = getattr(self_attn, "mla_attn", None) ++ if mla_attn is not None and hasattr(mla_attn, "skip_topk"): ++ mla_attn.skip_topk = skip ++ ++ def compact_topk_indices(self, slot_ids: torch.Tensor): ++ """Gather the top-k index rows at ``slot_ids`` to the front of the buffer.""" ++ num_slots = slot_ids.numel() ++ for layer in self.layers.values(): ++ mtp_block = getattr(layer, "mtp_block", None) ++ if mtp_block is not None: ++ self_attn = getattr(mtp_block, "self_attn", None) ++ if self_attn is not None: ++ mla_attn = getattr(self_attn, "mla_attn", None) ++ if mla_attn is not None and hasattr( ++ mla_attn, "topk_indices_buffer" ++ ): ++ topk_indices_buffer = mla_attn.topk_indices_buffer ++ topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] ++ ++ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: ++ return self.embed_tokens(input_ids) ++ ++ def forward( ++ self, ++ input_ids: torch.Tensor, ++ positions: torch.Tensor, ++ previous_hidden_states: torch.Tensor, ++ inputs_embeds: torch.Tensor | None = None, ++ spec_step_idx: int = 0, ++ ) -> torch.Tensor: ++ if inputs_embeds is None: ++ inputs_embeds = self.embed_tokens(input_ids) ++ current_step_idx = spec_step_idx % self.num_mtp_layers ++ return self.layers[str(self.mtp_start_layer_idx + current_step_idx)]( ++ input_ids, ++ positions, ++ previous_hidden_states, ++ inputs_embeds, ++ current_step_idx, ++ ) ++ ++ def compute_logits( ++ self, ++ hidden_states: torch.Tensor, ++ spec_step_idx: int = 0, ++ ) -> torch.Tensor: ++ current_step_idx = spec_step_idx % self.num_mtp_layers ++ mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] ++ logits = self.logits_processor( ++ mtp_layer.shared_head.head, mtp_layer.shared_head(hidden_states) ++ ) ++ return logits ++ ++ ++@support_torch_compile ++class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): ++ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): ++ super().__init__() ++ self.config = vllm_config.model_config.hf_config ++ self.quant_config = vllm_config.quant_config ++ self.model = DeepSeekMultiTokenPredictor( ++ vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ++ ) ++ # Set MoE hyperparameters ++ self.set_moe_parameters() ++ ++ def set_moe_parameters(self): ++ self.num_moe_layers = self.config.num_nextn_predict_layers ++ self.num_expert_groups = self.config.n_group ++ ++ self.moe_layers = [] ++ self.moe_mlp_layers = [] ++ example_moe = None ++ for layer in self.model.layers.values(): ++ assert isinstance(layer, DeepSeekMultiTokenPredictorLayer) ++ layer = layer.mtp_block ++ assert isinstance(layer, DeepseekV2DecoderLayer) ++ if isinstance(layer.mlp, DeepseekV2MoE): ++ example_moe = layer.mlp ++ self.moe_mlp_layers.append(layer.mlp) ++ self.moe_layers.append(layer.mlp.experts) ++ self.extract_moe_parameters(example_moe) ++ self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( ++ self.model.layers.values(), ++ DeepseekV2MoE, ++ "mtp_block.mlp", ++ ) ++ ++ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: ++ return self.model.embed_input_ids(input_ids) ++ ++ def forward( ++ self, ++ input_ids: torch.Tensor | None, ++ positions: torch.Tensor, ++ hidden_states: torch.Tensor, ++ intermediate_tensors: IntermediateTensors | None = None, ++ inputs_embeds: torch.Tensor | None = None, ++ spec_step_idx: int = 0, ++ ) -> torch.Tensor: ++ hidden_states = self.model( ++ input_ids, ++ positions, ++ hidden_states, ++ inputs_embeds, ++ spec_step_idx, ++ ) ++ return hidden_states ++ ++ def compute_logits( ++ self, ++ hidden_states: torch.Tensor, ++ spec_step_idx: int = 0, ++ ) -> torch.Tensor | None: ++ return self.model.compute_logits(hidden_states, spec_step_idx) ++ ++ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: ++ stacked_params_mapping = [ ++ ("gate_up_proj", "gate_proj", 0), ++ ("gate_up_proj", "up_proj", 1), ++ ("fused_qkv_a_proj", "q_a_proj", 0), ++ ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), ++ ] ++ ++ # Fused indexer wk + weights_proj (shard 0 = wk, shard 1 = weights_proj) ++ indexer_fused_mapping = [ ++ ("wk_weights_proj", "wk", 0), ++ ("wk_weights_proj", "weights_proj", 1), ++ ] ++ stacked_params_mapping.extend(indexer_fused_mapping) ++ ++ expert_params_mapping = fused_moe_make_expert_params_mapping( ++ self, ++ ckpt_gate_proj_name="gate_proj", ++ ckpt_down_proj_name="down_proj", ++ ckpt_up_proj_name="up_proj", ++ num_experts=self.config.n_routed_experts ++ + ( ++ self.config.n_shared_experts ++ if self.is_fused_shared_expert_enabled ++ else 0 ++ ), ++ ) ++ ++ pp_missing_layer_names = get_pp_missing_layer_names(self) ++ params_dict = dict(self.named_parameters()) ++ loaded_params: set[str] = set() ++ _pending_wk_fp8: dict = {} # FP8 indexer wk dequant buffer ++ for name, loaded_weight in weights: ++ if "rotary_emb.inv_freq" in name: ++ continue ++ spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) ++ if spec_layer is None: ++ continue ++ is_fusion_moe_shared_experts_layer = ( ++ self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) ++ ) ++ name = self._rewrite_spec_layer_name(spec_layer, name) ++ ++ if _try_load_fp8_indexer_wk( ++ name, ++ loaded_weight, ++ _pending_wk_fp8, ++ params_dict, ++ loaded_params, ++ pp_missing_layer_names, ++ ): ++ continue ++ ++ for param_name, weight_name, shard_id in stacked_params_mapping: ++ # Skip non-stacked layers and experts (experts handled below). ++ if weight_name not in name: ++ continue ++ # We have mlp.experts[0].gate_proj in the checkpoint. ++ # Since we handle the experts below in expert_params_mapping, ++ # we need to skip here BEFORE we update the name, otherwise ++ # name will be updated to mlp.experts[0].gate_up_proj, which ++ # will then be updated below in expert_params_mapping ++ # for mlp.experts[0].gate_gate_up_proj, which breaks load. ++ if ("mlp.experts." in name) and name not in params_dict: ++ continue ++ if is_fusion_moe_shared_experts_layer: ++ continue ++ name_mapped = name.replace(weight_name, param_name) ++ ++ # QKV fusion is optional, fall back to normal ++ # weight loading if it's not enabled ++ if ( ++ param_name == "fused_qkv_a_proj" ++ ) and name_mapped not in params_dict: ++ continue ++ else: ++ name = name_mapped ++ ++ # Skip loading extra bias for GPTQ models. ++ if name.endswith(".bias") and name not in params_dict: ++ continue ++ ++ param = params_dict[name] ++ weight_loader = param.weight_loader ++ weight_loader(param, loaded_weight, shard_id) ++ break ++ else: ++ # Special handling: when AITER fusion_shared_experts is enabled, ++ # checkpoints may provide a single widened shared_experts tensor ++ # without explicit expert indices ++ # (e.g. ...mlp.shared_experts.gate_proj.weight). ++ # For models with multiple shared experts, split that tensor ++ # evenly into per-shared-expert slices and load them into ++ # appended expert slots mlp.experts.{n_routed_experts + j}.* ++ # accordingly. ++ num_chunks = 1 ++ if is_fusion_moe_shared_experts_layer: ++ num_chunks = getattr(self.config, "n_shared_experts", 1) or 1 ++ # Determine split axis based on op type ++ # gate/up: ColumnParallel → split along dim 0 ++ # down: RowParallel → split along dim 1 ++ split_dim = ( ++ 1 ++ if ("down_proj.weight" in name and loaded_weight.ndim > 1) ++ else 0 ++ ) ++ total = loaded_weight.shape[split_dim] ++ assert total % num_chunks == 0, ( ++ f"Shared expert weight dim {total} " ++ f"not divisible by num_chunks {num_chunks}" ++ ) ++ chunk_size = total // num_chunks ++ ++ for j in range(num_chunks): ++ chunk_name = name ++ weight_to_load = loaded_weight ++ ++ if is_fusion_moe_shared_experts_layer: ++ chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size) ++ if loaded_weight.ndim == 1: ++ weight_to_load = loaded_weight[chunk_slice] ++ elif split_dim == 0: ++ weight_to_load = loaded_weight[chunk_slice, :] ++ else: ++ weight_to_load = loaded_weight[:, chunk_slice] ++ # Synthesize an expert-style name so expert mapping ++ # can route it ++ chunk_name = name.replace( ++ "mlp.shared_experts", ++ f"mlp.experts.{self.config.n_routed_experts + j}", ++ ) ++ ++ # Use expert_params_mapping to locate the destination ++ # param and delegate to its expert-aware weight_loader ++ # with expert_id. ++ is_expert_weight = False ++ for mapping in expert_params_mapping: ++ param_name, weight_name, expert_id, expert_shard_id = mapping ++ if weight_name not in chunk_name: ++ continue ++ ++ # Anyway, this is an expert weight and should not be ++ # attempted to load as other weights later ++ is_expert_weight = True ++ ++ # Do not modify `name` since the loop may continue here ++ # Instead, create a new variable ++ name_mapped = chunk_name.replace(weight_name, param_name) ++ ++ param = params_dict[name_mapped] ++ # We should ask the weight loader to return success or ++ # not here since otherwise we may skip experts with ++ # other available replicas. ++ weight_loader = typing.cast( ++ Callable[..., bool], param.weight_loader ++ ) ++ success = weight_loader( ++ param, ++ weight_to_load, ++ name_mapped, ++ shard_id=expert_shard_id, ++ expert_id=expert_id, ++ return_success=True, ++ ) ++ if success: ++ if not is_fusion_moe_shared_experts_layer: ++ name = name_mapped ++ else: ++ loaded_params.add(name_mapped) ++ break ++ else: ++ if is_expert_weight: ++ # We've checked that this is an expert weight ++ # However it's not mapped locally to this rank ++ # So we simply skip it ++ continue ++ ++ # Skip loading extra bias for GPTQ models. ++ if name.endswith(".bias") and name not in params_dict: ++ continue ++ ++ remapped_name = maybe_remap_kv_scale_name(name, params_dict) ++ if remapped_name is None: ++ continue ++ name = remapped_name ++ ++ # According to DeepSeek-V3 Technical Report, MTP modules ++ # shares embedding layer. We only load the first weights. ++ if ( ++ spec_layer != self.model.mtp_start_layer_idx ++ and ".layers" not in name ++ ): ++ continue ++ ++ param = params_dict[name] ++ weight_loader = getattr( ++ param, "weight_loader", default_weight_loader ++ ) ++ weight_loader(param, loaded_weight) ++ if not is_fusion_moe_shared_experts_layer: ++ loaded_params.add(name) ++ ++ # Validate that weights were loaded for each expected MTP layer. ++ loaded_layers: set[int] = set() ++ for param_name in loaded_params: ++ spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name) ++ if spec_layer is not None: ++ loaded_layers.add(spec_layer) ++ for layer_idx in range( ++ self.model.mtp_start_layer_idx, ++ self.model.mtp_start_layer_idx + self.model.num_mtp_layers, ++ ): ++ if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled(): ++ raise ValueError( ++ f"MTP speculative decoding layer {layer_idx} weights " ++ f"missing from checkpoint. The checkpoint may have " ++ f"been quantized without including the MTP layers. " ++ f"Use a checkpoint that includes MTP layer weights, " ++ f"or disable speculative decoding." ++ ) ++ ++ return loaded_params ++ ++ def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: ++ """ ++ Rewrite the weight name to match the format of the original model. ++ Add .mtp_block for modules in transformer layer block for spec layer ++ and rename shared layer weights to be top level. ++ """ ++ spec_layer_weight_names = [ ++ "embed_tokens", ++ "enorm", ++ "hnorm", ++ "eh_proj", ++ "shared_head", ++ ] ++ shared_weight_names = ["embed_tokens"] ++ spec_layer_weight = False ++ shared_weight = False ++ for weight_name in spec_layer_weight_names: ++ if weight_name in name: ++ spec_layer_weight = True ++ if weight_name in shared_weight_names: ++ shared_weight = True ++ break ++ if not spec_layer_weight: ++ # treat rest weights as weights for transformer layer block ++ name = name.replace( ++ f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block." ++ ) ++ elif shared_weight: ++ # treat shared weights as top level weights ++ name = name.replace(f"model.layers.{spec_layer}.", "model.") ++ return name +diff --git a/vllm/models/glm5next/model_state.py b/vllm/models/glm5next/model_state.py +index edeae144563c42df7bf55b7147ea7fdd3951fbf4..316332cb3f695d76705743d9b44629680fe582ab 100644 +--- a/vllm/models/glm5next/model_state.py ++++ b/vllm/models/glm5next/model_state.py +@@ -310,25 +310,13 @@ class Glm5NextModelState(MambaHybridModelState): + num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np) + + if self._align_mode: +- mamba_group_ids, _ = self._get_mamba_group_info(kv_cache_config) +- aligned_index_builders = [] +- for group_idx, group_id in enumerate(mamba_group_ids): +- for group in attn_groups[group_id]: +- builder = group.get_metadata_builder(0) +- if hasattr(builder, "mamba_aligned_state_indices"): +- aligned_index_builders.append((group_idx, builder)) +- if aligned_index_builders: +- ctx = self._ensure_align_ctx( +- kv_cache_config, +- mamba_group_ids, +- block_tables, +- ) +- all_group_indices = ctx.compute_aligned_state_indices( +- input_batch.seq_lens, +- num_reqs, +- ) +- for group_idx, builder in aligned_index_builders: +- builder.mamba_aligned_state_indices = all_group_indices[group_idx] ++ self._prepare_aligned_state_indices( ++ input_batch.seq_lens, ++ num_reqs, ++ attn_groups, ++ kv_cache_config, ++ block_tables, ++ ) + + model_metadata = Glm5NextAttnMetadata( + is_prefilling=is_prefilling, +diff --git a/vllm/models/glm5next/nvidia/mtp.py b/vllm/models/glm5next/nvidia/mtp.py +index 90bdb5b642144d2e5a29b9d2e7cb8d8c9277c71c..407ed1f21a72923fd000d564ab7d66f26015d538 100644 +--- a/vllm/models/glm5next/nvidia/mtp.py ++++ b/vllm/models/glm5next/nvidia/mtp.py +@@ -1,475 +1,515 @@ +-# SPDX-License-Identifier: Apache-2.0 +-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +-import typing +-from collections.abc import Callable, Iterable +- ++# SPDX-License-Identifier: Apache-2.0 ++# SPDX-FileCopyrightText: Copyright contributors to the vLLM project ++import typing ++from collections.abc import Callable, Iterable ++ + import torch + import torch.nn as nn + ++import vllm.envs as envs + from vllm.config import VllmConfig +-from vllm.model_executor.layers.fused_moe import ( +- fused_moe_make_expert_params_mapping, +-) +-from vllm.model_executor.layers.layernorm import RMSNorm +-from vllm.model_executor.layers.logits_processor import LogitsProcessor +-from vllm.model_executor.layers.vocab_parallel_embedding import ( +- VocabParallelEmbedding, +-) +-from vllm.model_executor.model_loader.weight_utils import ( +- default_weight_loader, +- maybe_remap_kv_scale_name, +-) +-from vllm.model_executor.models.deepseek_mtp import SharedHead +-from vllm.model_executor.models.deepseek_v2 import DeepseekV2MixtureOfExperts +-from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix +-from vllm.platforms import current_platform +-from vllm.sequence import IntermediateTensors +- +-from .model import ( +- GLM5NEXT_PACKED_MODULES_MAPPING, +- Glm5NextDecoderLayer, +- Glm5NextMoE, +- _try_load_fp8_attn_proj, +- _try_load_mxfp8_bf16_attn_proj, +- get_spec_layer_idx_from_weight_name, +-) +-from .pooled_indexer import Glm5NextPooledIndexer +- +- +-class Glm5NextMultiTokenPredictorLayer(nn.Module): +- def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: +- super().__init__() +- assert vllm_config.speculative_config is not None +- config = vllm_config.speculative_config.draft_model_config.hf_config +- self.config = config +- quant_config = vllm_config.quant_config +- +- self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) +- self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) +- self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) +- +- topk_tokens = config.index_topk +- kpool = getattr(config, "index_kpool", 1) or 1 +- buffer_width = topk_tokens + (kpool - 1 if kpool > 1 else 0) +- topk_indices_buffer = torch.empty( +- vllm_config.scheduler_config.max_num_batched_tokens, +- buffer_width, +- dtype=torch.int32, +- device=current_platform.device_type, +- ) +- pool_topk_indices_buffer = torch.empty( +- vllm_config.scheduler_config.max_num_batched_tokens, +- topk_tokens // kpool, +- dtype=torch.int32, +- device=current_platform.device_type, +- ) ++from vllm.model_executor.layers.fused_moe import ( ++ fused_moe_make_expert_params_mapping, ++) ++from vllm.model_executor.layers.layernorm import RMSNorm ++from vllm.model_executor.layers.logits_processor import LogitsProcessor ++from vllm.model_executor.layers.vocab_parallel_embedding import ( ++ VocabParallelEmbedding, ++) ++from vllm.model_executor.model_loader.weight_utils import ( ++ default_weight_loader, ++ maybe_remap_kv_scale_name, ++) ++from vllm.model_executor.models.deepseek_mtp import SharedHead ++from vllm.model_executor.models.deepseek_v2 import DeepseekV2MixtureOfExperts ++from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix ++from vllm.platforms import current_platform ++from vllm.sequence import IntermediateTensors ++ ++from .model import ( ++ GLM5NEXT_PACKED_MODULES_MAPPING, ++ Glm5NextDecoderLayer, ++ Glm5NextMoE, ++ _try_load_fp8_attn_proj, ++ _try_load_mxfp8_bf16_attn_proj, ++ get_spec_layer_idx_from_weight_name, ++) ++from .pooled_indexer import Glm5NextPooledIndexer ++ ++ ++class Glm5NextMultiTokenPredictorLayer(nn.Module): ++ def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: ++ super().__init__() ++ assert vllm_config.speculative_config is not None ++ config = vllm_config.speculative_config.draft_model_config.hf_config ++ self.config = config ++ quant_config = vllm_config.quant_config ++ ++ self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) ++ self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) ++ self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) ++ ++ topk_tokens = config.index_topk ++ kpool = getattr(config, "index_kpool", 1) or 1 ++ buffer_width = topk_tokens + (kpool - 1 if kpool > 1 else 0) ++ topk_indices_buffer = torch.empty( ++ vllm_config.scheduler_config.max_num_batched_tokens, ++ buffer_width, ++ dtype=torch.int32, ++ device=current_platform.device_type, ++ ) ++ pool_topk_indices_buffer = torch.empty( ++ vllm_config.scheduler_config.max_num_batched_tokens, ++ topk_tokens // kpool, ++ dtype=torch.int32, ++ device=current_platform.device_type, ++ ) + self.shared_head = SharedHead( +- config=config, prefix=prefix, quant_config=quant_config +- ) +- # MTP layers sit past the base model's hidden layers; parse the index +- # from the prefix (e.g. "...layers.32") so the decoder builds an MLA +- # (DSA) layer rather than KDA for the MTP path. +- layer_idx = int(prefix.rsplit(".", 1)[-1]) +- self.mtp_block = Glm5NextDecoderLayer( +- vllm_config=vllm_config, + config=config, +- layer_idx=layer_idx, + prefix=prefix, +- topk_indices_buffer=topk_indices_buffer, +- pool_topk_indices_buffer=pool_topk_indices_buffer, +- is_mtp_layer=True, +- ) +- +- def forward( +- self, +- input_ids: torch.Tensor, +- positions: torch.Tensor, +- previous_hidden_states: torch.Tensor, +- inputs_embeds: torch.Tensor | None = None, +- spec_step_index: int = 0, +- output_indices: torch.Tensor | None = None, +- ) -> torch.Tensor: +- assert inputs_embeds is not None +- eh_input = torch.cat( +- (self.enorm(inputs_embeds), self.hnorm(previous_hidden_states)), +- dim=-1, ++ quant_config=quant_config, ++ lm_head_quantization="nvfp4" if envs.VLLM_MTP_NVFP4_LM_HEAD else None, + ) +- hidden_states = self.eh_proj(eh_input) +- # Fuse the residual add and final RMSNorm. Glm5NextMoE already performs +- # its all-reduce, so no collective is needed here. The post-norm result +- # feeds both draft logits and the next recycled hidden state. +- hidden_states, residual, _, _ = self.mtp_block( +- positions=positions, +- hidden_states=hidden_states, +- residual=None, +- output_indices=output_indices, +- ) +- hidden_states, _ = self.shared_head.norm(hidden_states, residual=residual) +- return hidden_states, hidden_states +- +- +-class Glm5NextMultiTokenPredictor(nn.Module): +- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): +- super().__init__() +- config = vllm_config.model_config.hf_config +- self.mtp_start_layer_idx = config.num_hidden_layers +- self.num_mtp_layers = config.num_nextn_predict_layers +- self.layers = torch.nn.ModuleDict( +- { +- str(idx): Glm5NextMultiTokenPredictorLayer( +- vllm_config, f"{prefix}.layers.{idx}" +- ) +- for idx in range( +- self.mtp_start_layer_idx, +- self.mtp_start_layer_idx + self.num_mtp_layers, +- ) +- } +- ) +- self.embed_tokens = VocabParallelEmbedding( +- config.vocab_size, +- config.hidden_size, +- prefix=maybe_prefix(prefix, "embed_tokens"), +- ) +- # Plain list for the per-propose lookup: ModuleDict[str(...)] builds a +- # string and hashes it on every draft step. +- self._mtp_layers = list(self.layers.values()) +- self._prefill_output_indices: torch.Tensor | None = None +- self.logits_processor = LogitsProcessor(config.vocab_size) +- +- def update_max_model_len(self, max_model_len: int) -> None: +- for module in self.modules(): +- if isinstance(module, Glm5NextPooledIndexer): +- module.update_max_model_len(max_model_len) +- +- def set_skip_topk(self, skip: bool): +- # index_share_for_mtp_iteration: step 0 computes top-k, steps 1+ reuse. +- for layer in self.layers.values(): +- self_attn = getattr(layer.mtp_block, "self_attn", None) +- mla_attn = getattr(self_attn, "mla_attn", None) +- if mla_attn is not None and hasattr(mla_attn, "skip_topk"): +- mla_attn.skip_topk = skip +- +- def compact_topk_indices(self, slot_ids: torch.Tensor): +- """Gather the top-k index rows at ``slot_ids`` to the front of the buffer.""" +- num_slots = slot_ids.numel() +- for layer in self.layers.values(): +- self_attn = getattr(layer.mtp_block, "self_attn", None) +- mla_attn = getattr(self_attn, "mla_attn", None) +- if mla_attn is not None and hasattr(mla_attn, "topk_indices_buffer"): +- topk_indices_buffer = mla_attn.topk_indices_buffer +- topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] +- +- def snapshot_qsa_interval_starts(self) -> None: +- for layer in self.layers.values(): +- self_attn = getattr(layer.mtp_block, "self_attn", None) +- indexer = getattr(self_attn, "indexer", None) +- snapshot = getattr(indexer, "snapshot_speculative_interval_starts", None) +- if snapshot is not None: +- snapshot() +- +- def restore_qsa_interval_starts(self) -> None: +- for layer in self.layers.values(): +- self_attn = getattr(layer.mtp_block, "self_attn", None) +- indexer = getattr(self_attn, "indexer", None) +- restore = getattr(indexer, "restore_speculative_interval_starts", None) +- if restore is not None: +- restore() +- +- def set_prefill_output_indices(self, output_indices: torch.Tensor | None) -> None: +- """Select request-tail outputs after populating all MTP attention caches.""" +- self._prefill_output_indices = output_indices +- +- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: +- return self.embed_tokens(input_ids) +- +- def forward( +- self, +- input_ids: torch.Tensor, +- positions: torch.Tensor, +- previous_hidden_states: torch.Tensor, +- inputs_embeds: torch.Tensor | None = None, +- spec_step_idx: int = 0, +- ) -> torch.Tensor: +- if inputs_embeds is None: +- inputs_embeds = self.embed_tokens(input_ids) +- current_step_idx = spec_step_idx % self.num_mtp_layers +- return self._mtp_layers[current_step_idx]( +- input_ids, +- positions, +- previous_hidden_states, +- inputs_embeds, +- current_step_idx, +- self._prefill_output_indices, +- ) +- +- def compute_logits( +- self, +- hidden_states: torch.Tensor, +- spec_step_idx: int = 0, +- ) -> torch.Tensor: +- current_step_idx = spec_step_idx % self.num_mtp_layers +- mtp_layer = self._mtp_layers[current_step_idx] +- # hidden_states is already post-final-norm (produced in the layer +- # forward and recycled as-is); apply the LM head only, without a +- # second RMSNorm. +- return self.logits_processor(mtp_layer.shared_head.head, hidden_states) +- +- def get_top_tokens( +- self, +- hidden_states: torch.Tensor, +- spec_step_idx: int = 0, +- ) -> torch.Tensor: +- current_step_idx = spec_step_idx % self.num_mtp_layers +- mtp_layer = self._mtp_layers[current_step_idx] +- # Vocab-parallel argmax for the greedy draft: per-rank head projection +- # + local argmax + a [batch, 2*tp] (value, index) reduce, instead of +- # materializing and all-gathering full [N, vocab] logits per draft +- # step. Tie-breaking matches the full argmax (shards are contiguous +- # and rank-ordered, so the lowest-rank winner is the lowest global +- # index), so greedy draft tokens are unchanged. +- return self.logits_processor.get_top_tokens( +- mtp_layer.shared_head.head, hidden_states +- ) +- +- +-class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts): +- packed_modules_mapping = GLM5NEXT_PACKED_MODULES_MAPPING +- hf_to_vllm_mapper = WeightsMapper( +- orig_to_new_prefix={ +- "model.language_model.": "model.", +- "language_model.model.": "model.", +- } +- ) +- +- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): ++ # MTP layers sit past the base model's hidden layers; parse the index ++ # from the prefix (e.g. "...layers.32") so the decoder builds an MLA ++ # (DSA) layer rather than KDA for the MTP path. ++ layer_idx = int(prefix.rsplit(".", 1)[-1]) ++ self.mtp_block = Glm5NextDecoderLayer( ++ vllm_config=vllm_config, ++ config=config, ++ layer_idx=layer_idx, ++ prefix=prefix, ++ topk_indices_buffer=topk_indices_buffer, ++ pool_topk_indices_buffer=pool_topk_indices_buffer, ++ is_mtp_layer=True, ++ ) ++ ++ def forward( ++ self, ++ input_ids: torch.Tensor, ++ positions: torch.Tensor, ++ previous_hidden_states: torch.Tensor, ++ inputs_embeds: torch.Tensor | None = None, ++ spec_step_index: int = 0, ++ output_indices: torch.Tensor | None = None, ++ ) -> torch.Tensor: ++ assert inputs_embeds is not None ++ eh_input = torch.cat( ++ (self.enorm(inputs_embeds), self.hnorm(previous_hidden_states)), ++ dim=-1, ++ ) ++ hidden_states = self.eh_proj(eh_input) ++ # Fuse the residual add and final RMSNorm. Glm5NextMoE already performs ++ # its all-reduce, so no collective is needed here. The post-norm result ++ # feeds both draft logits and the next recycled hidden state. ++ hidden_states, residual, _, _ = self.mtp_block( ++ positions=positions, ++ hidden_states=hidden_states, ++ residual=None, ++ output_indices=output_indices, ++ ) ++ hidden_states, _ = self.shared_head.norm(hidden_states, residual=residual) ++ return hidden_states, hidden_states ++ ++ ++class Glm5NextMultiTokenPredictor(nn.Module): ++ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): ++ super().__init__() ++ config = vllm_config.model_config.hf_config ++ self.mtp_start_layer_idx = config.num_hidden_layers ++ self.num_mtp_layers = config.num_nextn_predict_layers ++ self.layers = torch.nn.ModuleDict( ++ { ++ str(idx): Glm5NextMultiTokenPredictorLayer( ++ vllm_config, f"{prefix}.layers.{idx}" ++ ) ++ for idx in range( ++ self.mtp_start_layer_idx, ++ self.mtp_start_layer_idx + self.num_mtp_layers, ++ ) ++ } ++ ) ++ self.embed_tokens = VocabParallelEmbedding( ++ config.vocab_size, ++ config.hidden_size, ++ prefix=maybe_prefix(prefix, "embed_tokens"), ++ ) ++ # Plain list for the per-propose lookup: ModuleDict[str(...)] builds a ++ # string and hashes it on every draft step. ++ self._mtp_layers = list(self.layers.values()) ++ self._prefill_output_indices: torch.Tensor | None = None ++ self.logits_processor = LogitsProcessor(config.vocab_size) ++ ++ def update_max_model_len(self, max_model_len: int) -> None: ++ for module in self.modules(): ++ if isinstance(module, Glm5NextPooledIndexer): ++ module.update_max_model_len(max_model_len) ++ ++ def set_skip_topk(self, skip: bool): ++ # index_share_for_mtp_iteration: step 0 computes top-k, steps 1+ reuse. ++ for layer in self.layers.values(): ++ self_attn = getattr(layer.mtp_block, "self_attn", None) ++ mla_attn = getattr(self_attn, "mla_attn", None) ++ if mla_attn is not None and hasattr(mla_attn, "skip_topk"): ++ mla_attn.skip_topk = skip ++ ++ def compact_topk_indices(self, slot_ids: torch.Tensor): ++ """Gather the top-k index rows at ``slot_ids`` to the front of the buffer.""" ++ num_slots = slot_ids.numel() ++ for layer in self.layers.values(): ++ self_attn = getattr(layer.mtp_block, "self_attn", None) ++ mla_attn = getattr(self_attn, "mla_attn", None) ++ if mla_attn is not None and hasattr(mla_attn, "topk_indices_buffer"): ++ topk_indices_buffer = mla_attn.topk_indices_buffer ++ topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] ++ ++ def snapshot_qsa_interval_starts(self) -> None: ++ for layer in self.layers.values(): ++ self_attn = getattr(layer.mtp_block, "self_attn", None) ++ indexer = getattr(self_attn, "indexer", None) ++ snapshot = getattr(indexer, "snapshot_speculative_interval_starts", None) ++ if snapshot is not None: ++ snapshot() ++ ++ def restore_qsa_interval_starts(self) -> None: ++ for layer in self.layers.values(): ++ self_attn = getattr(layer.mtp_block, "self_attn", None) ++ indexer = getattr(self_attn, "indexer", None) ++ restore = getattr(indexer, "restore_speculative_interval_starts", None) ++ if restore is not None: ++ restore() ++ ++ def set_prefill_output_indices(self, output_indices: torch.Tensor | None) -> None: ++ """Select request-tail outputs after populating all MTP attention caches.""" ++ self._prefill_output_indices = output_indices ++ ++ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: ++ return self.embed_tokens(input_ids) ++ ++ def forward( ++ self, ++ input_ids: torch.Tensor, ++ positions: torch.Tensor, ++ previous_hidden_states: torch.Tensor, ++ inputs_embeds: torch.Tensor | None = None, ++ spec_step_idx: int = 0, ++ ) -> torch.Tensor: ++ if inputs_embeds is None: ++ inputs_embeds = self.embed_tokens(input_ids) ++ current_step_idx = spec_step_idx % self.num_mtp_layers ++ return self._mtp_layers[current_step_idx]( ++ input_ids, ++ positions, ++ previous_hidden_states, ++ inputs_embeds, ++ current_step_idx, ++ self._prefill_output_indices, ++ ) ++ ++ def compute_logits( ++ self, ++ hidden_states: torch.Tensor, ++ spec_step_idx: int = 0, ++ ) -> torch.Tensor: ++ current_step_idx = spec_step_idx % self.num_mtp_layers ++ mtp_layer = self._mtp_layers[current_step_idx] ++ # hidden_states is already post-final-norm (produced in the layer ++ # forward and recycled as-is); apply the LM head only, without a ++ # second RMSNorm. ++ return self.logits_processor(mtp_layer.shared_head.head, hidden_states) ++ ++ def get_top_tokens( ++ self, ++ hidden_states: torch.Tensor, ++ spec_step_idx: int = 0, ++ ) -> torch.Tensor: ++ current_step_idx = spec_step_idx % self.num_mtp_layers ++ mtp_layer = self._mtp_layers[current_step_idx] ++ # Vocab-parallel argmax for the greedy draft: per-rank head projection ++ # + local argmax + a [batch, 2*tp] (value, index) reduce, instead of ++ # materializing and all-gathering full [N, vocab] logits per draft ++ # step. Tie-breaking matches the full argmax (shards are contiguous ++ # and rank-ordered, so the lowest-rank winner is the lowest global ++ # index), so greedy draft tokens are unchanged. ++ return self.logits_processor.get_top_tokens( ++ mtp_layer.shared_head.head, hidden_states ++ ) ++ ++ ++class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts): ++ packed_modules_mapping = GLM5NEXT_PACKED_MODULES_MAPPING ++ hf_to_vllm_mapper = WeightsMapper( ++ orig_to_new_prefix={ ++ "model.language_model.": "model.", ++ "language_model.model.": "model.", ++ } ++ ) ++ ++ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + self.config = vllm_config.model_config.hf_config + self.quant_config = vllm_config.quant_config +- self.checkpoint_weight_name_prefixes = self._checkpoint_weight_name_prefixes() ++ if envs.VLLM_MTP_NVFP4_LM_HEAD and self.config.tie_word_embeddings: ++ raise ValueError("NVFP4 proposal head requires untied word embeddings") + self.model = Glm5NextMultiTokenPredictor( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) ++ head = self.model._mtp_layers[0].shared_head.head ++ self.has_own_lm_head = head.runtime_lm_head_quantization == "nvfp4" ++ self.checkpoint_weight_name_prefixes = self._checkpoint_weight_name_prefixes() ++ if self.has_own_lm_head: ++ self.lm_head = head + self.set_moe_parameters() +- +- def _checkpoint_weight_name_prefixes(self) -> tuple[str, ...]: +- return tuple( ++ ++ def _checkpoint_weight_name_prefixes(self) -> tuple[str, ...]: ++ prefixes = tuple( + prefix +- for layer_idx in range( +- self.config.num_hidden_layers, +- self.config.num_hidden_layers + self.config.num_nextn_predict_layers, +- ) +- for prefix in ( +- f"model.language_model.layers.{layer_idx}.", +- f"language_model.model.layers.{layer_idx}.", +- f"model.layers.{layer_idx}.", ++ for layer_idx in range( ++ self.config.num_hidden_layers, ++ self.config.num_hidden_layers + self.config.num_nextn_predict_layers, ++ ) ++ for prefix in ( ++ f"model.language_model.layers.{layer_idx}.", ++ f"language_model.model.layers.{layer_idx}.", ++ f"model.layers.{layer_idx}.", + f"layers.{layer_idx}.", + ) + ) +- +- def set_moe_parameters(self): +- self.num_moe_layers = self.config.num_nextn_predict_layers +- self.num_expert_groups = self.config.n_group +- self.moe_layers = [] +- self.moe_mlp_layers = [] +- example_moe = None +- for layer in self.model.layers.values(): +- mlp = layer.mtp_block.mlp +- if isinstance(mlp, Glm5NextMoE): +- example_moe = mlp +- self.moe_mlp_layers.append(mlp) +- self.moe_layers.append(mlp.experts) +- self.extract_moe_parameters(example_moe) +- +- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: +- return self.model.embed_input_ids(input_ids) +- +- def update_max_model_len(self, max_model_len: int) -> None: +- self.model.update_max_model_len(max_model_len) +- +- def forward( +- self, +- input_ids: torch.Tensor | None, +- positions: torch.Tensor, +- hidden_states: torch.Tensor, +- intermediate_tensors: IntermediateTensors | None = None, +- inputs_embeds: torch.Tensor | None = None, +- spec_step_idx: int = 0, +- ) -> torch.Tensor: +- return self.model( +- input_ids, positions, hidden_states, inputs_embeds, spec_step_idx +- ) +- +- def compute_logits( +- self, +- hidden_states: torch.Tensor, +- spec_step_idx: int = 0, +- ) -> torch.Tensor | None: +- return self.model.compute_logits(hidden_states, spec_step_idx) +- +- def get_top_tokens( +- self, +- hidden_states: torch.Tensor, +- spec_step_idx: int = 0, +- ) -> torch.Tensor: +- # Greedy-draft path used when use_local_argmax_reduction is enabled: +- # vocab-parallel argmax, no full-vocab logits. +- return self.model.get_top_tokens(hidden_states, spec_step_idx) +- +- def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: +- spec_layer_weight_names = [ +- "embed_tokens", +- "enorm", +- "hnorm", +- "eh_proj", +- "shared_head", +- ] +- shared_weight_names = ["embed_tokens"] +- spec_layer_weight = False +- shared_weight = False +- for weight_name in spec_layer_weight_names: +- if weight_name in name: +- spec_layer_weight = True +- if weight_name in shared_weight_names: +- shared_weight = True +- break +- if not spec_layer_weight: +- name = name.replace( +- f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block." ++ if self.has_own_lm_head: ++ prefixes += ( ++ "lm_head.", ++ "model.lm_head.", ++ "model.language_model.lm_head.", ++ "language_model.lm_head.", + ) +- elif shared_weight: +- name = name.replace(f"model.layers.{spec_layer}.", "model.") +- return name +- +- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: +- stacked_params_mapping = [ +- ("gate_up_proj", "gate_proj", 0), +- ("gate_up_proj", "up_proj", 1), +- ("fused_qkv_a_proj", "q_a_proj", 0), +- ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), +- ] +- expert_params_mapping = fused_moe_make_expert_params_mapping( +- self, +- ckpt_gate_proj_name="gate_proj", +- ckpt_down_proj_name="down_proj", +- ckpt_up_proj_name="up_proj", +- num_experts=self.config.n_routed_experts, +- ) +- +- params_dict = dict(self.named_parameters()) +- loaded_params: set[str] = set() +- pending_attn_weights: dict = {} +- # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from +- # ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model. +- kv_a_pad_size = 0 +- if self.config.mla_nope and self.config.qk_rope_head_dim > 0: +- kv_a_pad_size = self.config.qk_rope_head_dim +- for name, loaded_weight in weights: +- if "rotary_emb.inv_freq" in name: +- continue +- # Multimodal (Glm5NextForConditionalGeneration) checkpoints prefix +- # the text-tower weights with "model.language_model."; the MTP head +- # is built as a text-only model (model.layers.*), so strip the +- # prefix to match. ++ return prefixes ++ ++ def set_moe_parameters(self): ++ self.num_moe_layers = self.config.num_nextn_predict_layers ++ self.num_expert_groups = self.config.n_group ++ self.moe_layers = [] ++ self.moe_mlp_layers = [] ++ example_moe = None ++ for layer in self.model.layers.values(): ++ mlp = layer.mtp_block.mlp ++ if isinstance(mlp, Glm5NextMoE): ++ example_moe = mlp ++ self.moe_mlp_layers.append(mlp) ++ self.moe_layers.append(mlp.experts) ++ self.extract_moe_parameters(example_moe) ++ ++ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: ++ return self.model.embed_input_ids(input_ids) ++ ++ def update_max_model_len(self, max_model_len: int) -> None: ++ self.model.update_max_model_len(max_model_len) ++ ++ def forward( ++ self, ++ input_ids: torch.Tensor | None, ++ positions: torch.Tensor, ++ hidden_states: torch.Tensor, ++ intermediate_tensors: IntermediateTensors | None = None, ++ inputs_embeds: torch.Tensor | None = None, ++ spec_step_idx: int = 0, ++ ) -> torch.Tensor: ++ return self.model( ++ input_ids, positions, hidden_states, inputs_embeds, spec_step_idx ++ ) ++ ++ def compute_logits( ++ self, ++ hidden_states: torch.Tensor, ++ spec_step_idx: int = 0, ++ ) -> torch.Tensor | None: ++ return self.model.compute_logits(hidden_states, spec_step_idx) ++ ++ def get_top_tokens( ++ self, ++ hidden_states: torch.Tensor, ++ spec_step_idx: int = 0, ++ ) -> torch.Tensor: ++ # Greedy-draft path used when use_local_argmax_reduction is enabled: ++ # vocab-parallel argmax, no full-vocab logits. ++ return self.model.get_top_tokens(hidden_states, spec_step_idx) ++ ++ def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: ++ spec_layer_weight_names = [ ++ "embed_tokens", ++ "enorm", ++ "hnorm", ++ "eh_proj", ++ "shared_head", ++ ] ++ shared_weight_names = ["embed_tokens"] ++ spec_layer_weight = False ++ shared_weight = False ++ for weight_name in spec_layer_weight_names: ++ if weight_name in name: ++ spec_layer_weight = True ++ if weight_name in shared_weight_names: ++ shared_weight = True ++ break ++ if not spec_layer_weight: ++ name = name.replace( ++ f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block." ++ ) ++ elif shared_weight: ++ name = name.replace(f"model.layers.{spec_layer}.", "model.") ++ return name ++ ++ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: ++ stacked_params_mapping = [ ++ ("gate_up_proj", "gate_proj", 0), ++ ("gate_up_proj", "up_proj", 1), ++ ("fused_qkv_a_proj", "q_a_proj", 0), ++ ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), ++ ] ++ expert_params_mapping = fused_moe_make_expert_params_mapping( ++ self, ++ ckpt_gate_proj_name="gate_proj", ++ ckpt_down_proj_name="down_proj", ++ ckpt_up_proj_name="up_proj", ++ num_experts=self.config.n_routed_experts, ++ ) ++ ++ params_dict = dict(self.named_parameters()) ++ loaded_params: set[str] = set() ++ pending_attn_weights: dict = {} ++ # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from ++ # ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model. ++ kv_a_pad_size = 0 ++ if self.config.mla_nope and self.config.qk_rope_head_dim > 0: ++ kv_a_pad_size = self.config.qk_rope_head_dim ++ for name, loaded_weight in weights: ++ if "rotary_emb.inv_freq" in name: ++ continue ++ # Multimodal (Glm5NextForConditionalGeneration) checkpoints prefix ++ # the text-tower weights with "model.language_model."; the MTP head ++ # is built as a text-only model (model.layers.*), so strip the ++ # prefix to match. + if name.startswith("model.language_model."): + name = name.replace("model.language_model.", "model.", 1) +- spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) +- if spec_layer is None: +- continue +- name = self._rewrite_spec_layer_name(spec_layer, name) +- +- if _try_load_mxfp8_bf16_attn_proj( +- name, +- loaded_weight, +- pending_attn_weights, +- params_dict, +- loaded_params, ++ if name in ( ++ "lm_head.weight", ++ "model.lm_head.weight", ++ "language_model.lm_head.weight", + ): ++ if self.has_own_lm_head: ++ for layer_idx in self.model.layers: ++ head_name = f"model.layers.{layer_idx}.shared_head.head.weight" ++ if head_name not in loaded_params: ++ param = params_dict[head_name] ++ param.weight_loader(param, loaded_weight) ++ loaded_params.add(head_name) + continue +- +- # Dequantize legacy block-FP8 projections kept in BF16. +- if _try_load_fp8_attn_proj( +- name, +- loaded_weight, +- pending_attn_weights, +- params_dict, +- loaded_params, +- kv_a_pad_size, +- ): +- continue +- +- for param_name, weight_name, shard_id in stacked_params_mapping: +- if weight_name not in name: +- continue +- if ("mlp.experts." in name) and name not in params_dict: +- continue +- name_mapped = name.replace(weight_name, param_name) +- if ( +- param_name == "fused_qkv_a_proj" +- ) and name_mapped not in params_dict: +- continue +- else: +- name = name_mapped +- if name.endswith(".bias") and name not in params_dict: +- continue +- param = params_dict[name] +- weight_loader = param.weight_loader +- weight_loader(param, loaded_weight, shard_id) +- break +- else: +- is_expert_weight = False +- for mapping in expert_params_mapping: +- param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment] +- if weight_name not in name: +- continue +- is_expert_weight = True +- name_mapped = name.replace(weight_name, param_name) +- param = params_dict[name_mapped] +- weight_loader = typing.cast( +- Callable[..., bool], param.weight_loader +- ) +- success = weight_loader( +- param, +- loaded_weight, +- name_mapped, +- shard_id=shard_id, +- expert_id=expert_id, +- return_success=True, +- ) +- if success: +- name = name_mapped +- break +- else: +- if is_expert_weight: +- continue +- if name.endswith(".bias") and name not in params_dict: +- continue +- name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment] +- if name is None: +- continue +- if ( +- spec_layer != self.model.mtp_start_layer_idx +- and ".layers" not in name +- ): +- continue +- param = params_dict[name] +- weight_loader = getattr( +- param, "weight_loader", default_weight_loader +- ) +- weight_loader(param, loaded_weight) +- loaded_params.add(name) +- ++ spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) ++ if spec_layer is None: ++ continue ++ name = self._rewrite_spec_layer_name(spec_layer, name) ++ ++ if _try_load_mxfp8_bf16_attn_proj( ++ name, ++ loaded_weight, ++ pending_attn_weights, ++ params_dict, ++ loaded_params, ++ ): ++ continue ++ ++ # Dequantize legacy block-FP8 projections kept in BF16. ++ if _try_load_fp8_attn_proj( ++ name, ++ loaded_weight, ++ pending_attn_weights, ++ params_dict, ++ loaded_params, ++ kv_a_pad_size, ++ ): ++ continue ++ ++ for param_name, weight_name, shard_id in stacked_params_mapping: ++ if weight_name not in name: ++ continue ++ if ("mlp.experts." in name) and name not in params_dict: ++ continue ++ name_mapped = name.replace(weight_name, param_name) ++ if ( ++ param_name == "fused_qkv_a_proj" ++ ) and name_mapped not in params_dict: ++ continue ++ else: ++ name = name_mapped ++ if name.endswith(".bias") and name not in params_dict: ++ continue ++ param = params_dict[name] ++ weight_loader = param.weight_loader ++ weight_loader(param, loaded_weight, shard_id) ++ break ++ else: ++ is_expert_weight = False ++ for mapping in expert_params_mapping: ++ param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment] ++ if weight_name not in name: ++ continue ++ is_expert_weight = True ++ name_mapped = name.replace(weight_name, param_name) ++ param = params_dict[name_mapped] ++ weight_loader = typing.cast( ++ Callable[..., bool], param.weight_loader ++ ) ++ success = weight_loader( ++ param, ++ loaded_weight, ++ name_mapped, ++ shard_id=shard_id, ++ expert_id=expert_id, ++ return_success=True, ++ ) ++ if success: ++ name = name_mapped ++ break ++ else: ++ if is_expert_weight: ++ continue ++ if name.endswith(".bias") and name not in params_dict: ++ continue ++ name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment] ++ if name is None: ++ continue ++ if ( ++ spec_layer != self.model.mtp_start_layer_idx ++ and ".layers" not in name ++ ): ++ continue ++ param = params_dict[name] ++ weight_loader = getattr( ++ param, "weight_loader", default_weight_loader ++ ) ++ weight_loader(param, loaded_weight) ++ loaded_params.add(name) ++ + loaded_layers: set[int] = set() + for param_name in loaded_params: ++ if param_name.endswith(".shared_head.head.weight"): ++ continue + spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name) +- if spec_layer is not None: +- loaded_layers.add(spec_layer) +- for layer_idx in range( +- self.model.mtp_start_layer_idx, +- self.model.mtp_start_layer_idx + self.model.num_mtp_layers, ++ if spec_layer is not None: ++ loaded_layers.add(spec_layer) ++ for layer_idx in range( ++ self.model.mtp_start_layer_idx, ++ self.model.mtp_start_layer_idx + self.model.num_mtp_layers, + ): ++ if self.has_own_lm_head: ++ head_name = f"model.layers.{layer_idx}.shared_head.head.weight" ++ if head_name not in loaded_params: ++ raise ValueError( ++ f"NVFP4 MTP head {layer_idx} requires an unquantized " ++ "proposal head or target lm_head.weight in the checkpoint." ++ ) + if layer_idx not in loaded_layers: +- raise ValueError( +- f"MTP speculative decoding layer {layer_idx} weights " +- f"missing from checkpoint." +- ) +- return loaded_params ++ raise ValueError( ++ f"MTP speculative decoding layer {layer_idx} weights " ++ f"missing from checkpoint." ++ ) ++ return loaded_params +diff --git a/vllm/utils/b12x.py b/vllm/utils/b12x.py +index aef960a475a5a8cee7f7ca27d70735af55265ba6..e6c794a84a4f48ed277de3e04203e71bae8827d4 100644 +--- a/vllm/utils/b12x.py ++++ b/vllm/utils/b12x.py +@@ -7,10 +7,12 @@ import importlib.util + from collections.abc import Callable, Hashable, Iterable + from dataclasses import dataclass, fields, is_dataclass + from types import ModuleType +-from typing import Any ++from typing import Any, Literal + + import torch + ++import vllm.envs as envs ++ + + @dataclass(frozen=True) + class B12xWarmupUnit: +@@ -19,6 +21,12 @@ class B12xWarmupUnit: + compile: Callable[[], None] + + ++def get_b12x_dense_activation_mode(recipe: Literal["nvfp4", "mxfp8"]) -> str: ++ """Resolve the dense precision override once when loading a layer.""" ++ override = getattr(envs, f"VLLM_B12X_{recipe.upper()}_ACTIVATION_MODE") ++ return override if override is not None else envs.VLLM_B12X_DENSE_ACTIVATION_MODE ++ ++ + _HAS_B12X = importlib.util.find_spec("b12x") is not None + + +diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py +index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae9275d456 100644 +--- a/vllm/v1/attention/backends/gdn_attn.py ++++ b/vllm/v1/attention/backends/gdn_attn.py +@@ -2,11 +2,13 @@ + # SPDX-FileCopyrightText: Copyright contributors to the vLLM project + """Backend for GatedDeltaNet attention.""" + ++from copy import copy + from dataclasses import dataclass, replace + from typing import Literal + + import torch + ++import vllm.envs as envs + from vllm.config import VllmConfig + from vllm.utils.torch_utils import async_tensor_h2d + from vllm.v1.attention.backend import ( +@@ -101,6 +103,7 @@ class GDNAttentionMetadata: + seq_lens: torch.Tensor | None = None + + prefill_checkpoint: GDNPrefillCheckpointMetadata | None = None ++ is_uniform_spec_decode: bool = False + + + class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]): +@@ -108,7 +111,9 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] + _cudagraph_support = AttentionCGSupport.UNIFORM_BATCH + supports_update_block_table: bool = True + ++ # Runner-owned stable storage, with NULL_BLOCK_ID in padded request rows. + mamba_aligned_state_indices: torch.Tensor | None = None ++ mamba_spec_accepted_tokens: torch.Tensor | None = None + + reorder_batch_threshold: int = 1 + +@@ -190,6 +195,84 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] + dtype=torch.int32, + device=device, + ) ++ self._decode_state_indices_source: torch.Tensor | None = None ++ self._decode_state_indices_view: torch.Tensor | None = None ++ self._reuse_spec_decode_inputs = envs.VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH ++ self._uniform_spec_masks = torch.ones( ++ self.decode_cudagraph_max_bs, dtype=torch.bool, device=device ++ ) ++ self._uniform_spec_masks_cpu = torch.ones( ++ self.decode_cudagraph_max_bs, dtype=torch.bool ++ ) ++ self._uniform_spec_tokens = torch.arange( ++ self.decode_cudagraph_max_bs, dtype=torch.int32, device=device ++ ) ++ self._uniform_spec_query_start = torch.arange( ++ self.decode_cudagraph_max_bs + 1, dtype=torch.int32, device=device ++ ) * (self.num_spec + 1) ++ self._spec_state_indices_source: torch.Tensor | None = None ++ self._spec_state_indices_view: torch.Tensor | None = None ++ ++ def _can_reuse_spec_inputs( ++ self, ++ m: CommonAttentionMetadata, ++ num_accepted_tokens: torch.Tensor | None, ++ num_decode_draft_tokens_cpu: torch.Tensor | None, ++ ) -> bool: ++ return ( ++ self._reuse_spec_decode_inputs ++ and self.use_spec_decode ++ and self.use_full_cuda_graph ++ and self.vllm_config.cache_config.mamba_cache_mode == "align" ++ and self.mamba_aligned_state_indices is not None ++ and self.mamba_spec_accepted_tokens is not None ++ and num_accepted_tokens is not None ++ and num_decode_draft_tokens_cpu is not None ++ and 0 < m.num_actual_tokens <= self.decode_cudagraph_max_bs ++ and m.num_actual_tokens == m.num_reqs * (self.num_spec + 1) ++ and bool(torch.all(num_decode_draft_tokens_cpu == self.num_spec)) ++ and bool(torch.all(torch.diff(m.query_start_loc_cpu) == self.num_spec + 1)) ++ and (m.is_prefilling is None or not bool(torch.any(m.is_prefilling))) ++ ) ++ ++ def _get_spec_state_indices_view(self, num_reqs: int) -> torch.Tensor: ++ source = self.mamba_aligned_state_indices ++ assert source is not None ++ if ( ++ self._spec_state_indices_source is not source ++ or self._spec_state_indices_view is None ++ or self._spec_state_indices_view.shape[0] != num_reqs ++ ): ++ self._spec_state_indices_source = source ++ self._spec_state_indices_view = source[:num_reqs, : self.num_spec + 1] ++ return self._spec_state_indices_view ++ ++ def _build_uniform_spec_decode( ++ self, m: CommonAttentionMetadata, num_accepted_tokens: torch.Tensor ++ ) -> GDNAttentionMetadata: ++ num_reqs = m.num_reqs ++ assert self.mamba_spec_accepted_tokens is not None ++ accepted = self.mamba_spec_accepted_tokens[:num_reqs] ++ accepted.copy_(num_accepted_tokens[:num_reqs], non_blocking=True) ++ return GDNAttentionMetadata( ++ num_prefills=0, ++ num_prefill_tokens=0, ++ num_decodes=0, ++ num_decode_tokens=0, ++ num_spec_decodes=num_reqs, ++ num_spec_decode_tokens=m.num_actual_tokens, ++ num_actual_tokens=m.num_actual_tokens, ++ spec_query_start_loc=self._uniform_spec_query_start[: num_reqs + 1], ++ spec_state_indices_tensor=self._get_spec_state_indices_view(num_reqs), ++ spec_sequence_masks=self._uniform_spec_masks[:num_reqs], ++ spec_sequence_masks_cpu=self._uniform_spec_masks_cpu[:num_reqs], ++ spec_token_indx=self._uniform_spec_tokens[: m.num_actual_tokens], ++ non_spec_token_indx=self._uniform_spec_tokens[:0], ++ num_accepted_tokens=accepted, ++ num_reqs=num_reqs, ++ seq_lens=m.seq_lens, ++ is_uniform_spec_decode=True, ++ ) + + def _get_state_indices( + self, +@@ -209,6 +292,13 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] + self.vllm_config.cache_config.mamba_cache_mode, + ) + ++ def _can_reuse_decode_inputs(self) -> bool: ++ return ( ++ not self.use_spec_decode ++ and self.vllm_config.cache_config.mamba_cache_mode == "align" ++ and self.mamba_aligned_state_indices is not None ++ ) ++ + def _build_chunk_metadata( + self, + prefill_query_start_loc: torch.Tensor, +@@ -260,6 +350,11 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] + fast_build: bool = False, + ) -> GDNAttentionMetadata: + m = common_attn_metadata ++ if self._can_reuse_spec_inputs( ++ m, num_accepted_tokens, num_decode_draft_tokens_cpu ++ ): ++ assert num_accepted_tokens is not None ++ return self._build_uniform_spec_decode(m, num_accepted_tokens) + + query_start_loc = m.query_start_loc + query_start_loc_cpu = m.query_start_loc_cpu +@@ -599,6 +694,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] + and num_prefills == 0 + and num_spec_decodes == 0 + and num_decodes <= self.decode_cudagraph_max_bs ++ and not self._can_reuse_decode_inputs() + ): + self.non_spec_state_indices_tensor[:num_decodes].copy_( + non_spec_state_indices_tensor, non_blocking=True +@@ -657,6 +753,41 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] + assert metadata.num_reqs > 0 + assert metadata.seq_lens is not None + ++ if ( ++ metadata.is_uniform_spec_decode ++ and self._reuse_spec_decode_inputs ++ and self.mamba_aligned_state_indices is not None ++ and self.mamba_spec_accepted_tokens is not None ++ ): ++ updated = copy(metadata) ++ updated.spec_state_indices_tensor = self._get_spec_state_indices_view( ++ metadata.num_reqs ++ ) ++ accepted = self.mamba_spec_accepted_tokens[: metadata.num_reqs] ++ assert metadata.num_accepted_tokens is not None ++ if accepted.data_ptr() != metadata.num_accepted_tokens.data_ptr(): ++ accepted.copy_(metadata.num_accepted_tokens, non_blocking=True) ++ updated.num_accepted_tokens = accepted ++ return updated ++ ++ if ( ++ metadata.num_prefills == 0 ++ and metadata.num_spec_decodes == 0 ++ and self._can_reuse_decode_inputs() ++ ): ++ source = self.mamba_aligned_state_indices ++ assert source is not None ++ if ( ++ self._decode_state_indices_source is not source ++ or self._decode_state_indices_view is None ++ or self._decode_state_indices_view.shape[0] != metadata.num_reqs ++ ): ++ self._decode_state_indices_source = source ++ self._decode_state_indices_view = source[: metadata.num_reqs, 0] ++ updated = copy(metadata) ++ updated.non_spec_state_indices_tensor = self._decode_state_indices_view ++ return updated ++ + state_indices = self._get_state_indices( + blk_table, + metadata.seq_lens, +@@ -753,6 +884,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] + and metadata.num_prefills == 0 + and metadata.num_spec_decodes == 0 + and metadata.num_decodes <= self.decode_cudagraph_max_bs ++ and not self._can_reuse_decode_inputs() + ): + self.non_spec_state_indices_tensor[: metadata.num_decodes].copy_( + non_spec_state_indices[: metadata.num_decodes], non_blocking=True +diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py +index e225c78ffe1989bffaf33a4b3a4da3feab19bf75..cbecda1e6c33b2c0a7298eb01ed38d2bb11b1ed7 100644 +--- a/vllm/v1/worker/gpu/attn_utils.py ++++ b/vllm/v1/worker/gpu/attn_utils.py +@@ -293,43 +293,11 @@ def build_attn_metadata( + attn_metadata: dict[str, Any] = {} + cached_attn_metadata: dict[tuple[KVCacheSpec, type], Any] = {} + num_kv_cache_groups = len(kv_cache_config.kv_cache_groups) ++ group_slot_mappings = slot_mappings[:num_kv_cache_groups].unbind(0) + for i in range(num_kv_cache_groups): + block_table = block_tables[i] +- slot_mapping = slot_mappings[i] +- # Per-group causal for hybrid drafters (mixed SWA/full attention). +- group_causal = ( +- causal if isinstance(causal, (bool, torch.Tensor)) else causal.get(i, True) +- ) +- +- common_attn_metadata_extra_kwargs = ( +- model_specific_attn_metadata.get_extra_common_attn_kwargs(i, num_reqs) +- if model_specific_attn_metadata is not None +- else {} +- ) +- # Model-specific metadata (e.g. Mamba hybrid) may supply its own +- # padding-aware is_prefilling, which takes precedence over the default. +- group_is_prefilling = common_attn_metadata_extra_kwargs.pop( +- "is_prefilling", is_prefilling +- ) +- common_attn_metadata = CommonAttentionMetadata( +- query_start_loc=query_start_loc_gpu, +- query_start_loc_cpu=query_start_loc_cpu, +- seq_lens=seq_lens, +- seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, +- max_seq_len=max_seq_len, +- num_reqs=num_reqs, +- num_actual_tokens=num_tokens, +- max_query_len=max_query_len, +- block_table_tensor=block_table, +- slot_mapping=slot_mapping, +- causal=group_causal, +- dcp_local_seq_lens=dcp_local_seq_lens, +- positions=positions, +- is_prefilling=group_is_prefilling, +- mm_req_doc_ranges=mm_req_doc_ranges, +- rswa_prefix_lens=rswa_prefix_lens, +- **common_attn_metadata_extra_kwargs, +- ) ++ slot_mapping = group_slot_mappings[i] ++ common_attn_metadata = None + + for attn_group in attn_groups[i]: + attn_metadata_builder = attn_group.get_metadata_builder(0) +@@ -337,35 +305,74 @@ def build_attn_metadata( + if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs): + kv_cache_spec = kv_cache_spec.kv_cache_specs[attn_group.layer_names[0]] + cache_key = (kv_cache_spec, type(attn_metadata_builder)) +- if for_cudagraph_capture: +- metadata = attn_metadata_builder.build_for_cudagraph_capture( +- common_attn_metadata +- ) +- elif ( +- cache_key in cached_attn_metadata ++ if ( ++ not for_cudagraph_capture ++ and cache_key in cached_attn_metadata + and attn_metadata_builder.supports_update_block_table + ): + metadata = attn_metadata_builder.update_block_table( +- cached_attn_metadata[cache_key], +- common_attn_metadata.block_table_tensor, +- common_attn_metadata.slot_mapping, ++ cached_attn_metadata[cache_key], block_table, slot_mapping + ) + else: +- attn_metadata_extra_kwargs = ( +- model_specific_attn_metadata.get_extra_attn_kwargs( +- attn_metadata_builder, +- num_reqs, ++ if common_attn_metadata is None: ++ # Per-group causal for hybrid drafters (mixed SWA/full attention). ++ group_causal = ( ++ causal ++ if isinstance(causal, (bool, torch.Tensor)) ++ else causal.get(i, True) + ) +- if model_specific_attn_metadata is not None +- else {} +- ) +- metadata = attn_metadata_builder.build( +- common_prefix_len=0, +- common_attn_metadata=common_attn_metadata, +- **attn_metadata_extra_kwargs, +- ) +- if attn_metadata_builder.supports_update_block_table: +- cached_attn_metadata[cache_key] = metadata ++ ++ common_attn_metadata_extra_kwargs = ( ++ model_specific_attn_metadata.get_extra_common_attn_kwargs( ++ i, num_reqs ++ ) ++ if model_specific_attn_metadata is not None ++ else {} ++ ) ++ # Model-specific padding takes precedence over the default. ++ group_is_prefilling = common_attn_metadata_extra_kwargs.pop( ++ "is_prefilling", is_prefilling ++ ) ++ common_attn_metadata = CommonAttentionMetadata( ++ query_start_loc=query_start_loc_gpu, ++ query_start_loc_cpu=query_start_loc_cpu, ++ seq_lens=seq_lens, ++ seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, ++ max_seq_len=max_seq_len, ++ num_reqs=num_reqs, ++ num_actual_tokens=num_tokens, ++ max_query_len=max_query_len, ++ block_table_tensor=block_table, ++ slot_mapping=slot_mapping, ++ causal=group_causal, ++ dcp_local_seq_lens=dcp_local_seq_lens, ++ positions=positions, ++ is_prefilling=group_is_prefilling, ++ mm_req_doc_ranges=mm_req_doc_ranges, ++ rswa_prefix_lens=rswa_prefix_lens, ++ **common_attn_metadata_extra_kwargs, ++ ) ++ ++ if for_cudagraph_capture: ++ metadata = attn_metadata_builder.build_for_cudagraph_capture( ++ common_attn_metadata ++ ) ++ else: ++ attn_metadata_extra_kwargs = ( ++ model_specific_attn_metadata.get_extra_attn_kwargs( ++ attn_metadata_builder, ++ num_reqs, ++ ) ++ if model_specific_attn_metadata is not None ++ else {} ++ ) ++ metadata = attn_metadata_builder.build( ++ common_prefix_len=0, ++ common_attn_metadata=common_attn_metadata, ++ **attn_metadata_extra_kwargs, ++ ) ++ if attn_metadata_builder.supports_update_block_table: ++ cached_attn_metadata[cache_key] = metadata + for layer_name in attn_group.layer_names: + attn_metadata[layer_name] = metadata + return attn_metadata +diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +index 7f4219288bb402703200e0edfbe55d43ff7b1dd0..532ce32e16f740f069731720ba0840312a96137d 100644 +--- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py ++++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +@@ -7,6 +7,7 @@ import numpy as np + import torch + import torch.nn as nn + ++from vllm import envs + from vllm.config import VllmConfig + from vllm.config.compilation import CUDAGraphMode + from vllm.triton_utils import tl, triton +@@ -84,6 +85,11 @@ class MambaHybridModelState(DefaultModelState): + self.num_accepted_tokens_gpu = torch.ones( + self.max_num_reqs, dtype=torch.int32, device=self.device + ) ++ self._gdn_spec_accepted_tokens = ( ++ torch.ones_like(self.num_accepted_tokens_gpu) ++ if envs.VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH ++ else None ++ ) + # Pre-copy "align" prefix-cache state (V2). The migration of each + # request's mamba state across block boundaries runs as a fused GPU + # kernel reusing the postprocess copy machinery, so the per-step src +@@ -106,6 +112,9 @@ class MambaHybridModelState(DefaultModelState): + self._mamba_group_ids: list[int] = [] + self._mamba_spec: MambaSpec | None = None + self._mamba_copy_funcs_by_type: MambaStateCopyFuncsByType | None = None ++ self._aligned_metadata_groups: list[list[AttentionGroup]] | None = None ++ self._aligned_metadata_builders: list[tuple[int, Any]] = [] ++ self._aligned_metadata_ctx: MambaSpecDecodeGPUContext | None = None + + def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: + super().add_request(req_index, new_req_data) +@@ -122,6 +131,9 @@ class MambaHybridModelState(DefaultModelState): + if self._align_mode: + self._mamba_ctx = None + self._mamba_copy_funcs_by_type = None ++ self._aligned_metadata_ctx = None ++ self._aligned_metadata_groups = None ++ self._aligned_metadata_builders = [] + if self.recoverssm is not None: + self.recoverssm.reset() + +@@ -171,6 +183,40 @@ class MambaHybridModelState(DefaultModelState): + ) + return ctx + ++ def _prepare_aligned_state_indices( ++ self, ++ seq_lens: torch.Tensor, ++ num_reqs: int, ++ attn_groups: list[list[AttentionGroup]], ++ kv_cache_config: KVCacheConfig, ++ block_tables: tuple[torch.Tensor, ...], ++ ) -> None: ++ mamba_group_ids, _ = self._get_mamba_group_info(kv_cache_config) ++ if self._aligned_metadata_groups is not attn_groups: ++ self._aligned_metadata_builders = [] ++ for group_idx, group_id in enumerate(mamba_group_ids): ++ for group in attn_groups[group_id]: ++ builder = group.get_metadata_builder(0) ++ if hasattr(builder, "mamba_aligned_state_indices"): ++ self._aligned_metadata_builders.append((group_idx, builder)) ++ if hasattr(builder, "mamba_spec_accepted_tokens"): ++ builder.mamba_spec_accepted_tokens = ( ++ self._gdn_spec_accepted_tokens ++ ) ++ self._aligned_metadata_groups = attn_groups ++ self._aligned_metadata_ctx = None ++ if not self._aligned_metadata_builders: ++ return ++ ++ ctx = self._ensure_align_ctx(kv_cache_config, mamba_group_ids, block_tables) ++ if self._aligned_metadata_ctx is not ctx: ++ assert ctx.aligned_state_indices is not None ++ group_views = ctx.aligned_state_indices.unbind(0) ++ for group_idx, builder in self._aligned_metadata_builders: ++ builder.mamba_aligned_state_indices = group_views[group_idx] ++ self._aligned_metadata_ctx = ctx ++ ctx.compute_aligned_state_indices(seq_lens, num_reqs) ++ + def preprocess_state( + self, + input_batch: InputBatch, +@@ -277,22 +323,13 @@ class MambaHybridModelState(DefaultModelState): + num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np) + + if self._align_mode: +- mamba_group_ids, _ = self._get_mamba_group_info(kv_cache_config) +- aligned_index_builders = [] +- for group_idx, group_id in enumerate(mamba_group_ids): +- for group in attn_groups[group_id]: +- builder = group.get_metadata_builder(0) +- if hasattr(builder, "mamba_aligned_state_indices"): +- aligned_index_builders.append((group_idx, builder)) +- if aligned_index_builders: +- ctx = self._ensure_align_ctx( +- kv_cache_config, mamba_group_ids, block_tables +- ) +- all_group_indices = ctx.compute_aligned_state_indices( +- input_batch.seq_lens, num_reqs +- ) +- for group_idx, builder in aligned_index_builders: +- builder.mamba_aligned_state_indices = all_group_indices[group_idx] ++ self._prepare_aligned_state_indices( ++ input_batch.seq_lens, ++ num_reqs, ++ attn_groups, ++ kv_cache_config, ++ block_tables, ++ ) + + mamba_attn_metadata = MambaHybridAttnMetadata( + is_prefilling=is_prefilling, +diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py +index 4857795fab43ff2daedec18d6acec301a376f825..3e92514f68a6a19a3bb1f91a631f429450226414 100644 +--- a/vllm/v1/worker/mamba_utils.py ++++ b/vllm/v1/worker/mamba_utils.py +@@ -85,8 +85,10 @@ def get_aligned_state_indices_multi_group_kernel( + mask=( + valid_group[:, None, None] + & valid_row[None, :, None] ++ & (seq_lens[None, :, None] > 0) + & valid_state_slot[None, None, :] + ), ++ other=-1, + ) + tl.store( + state_indices_ptr diff --git a/runtime/glm53-spark-mtp3-mesh/image-receipt.json b/runtime/glm53-spark-mtp3-mesh/image-receipt.json index a54d72c9..8a9e21e2 100644 --- a/runtime/glm53-spark-mtp3-mesh/image-receipt.json +++ b/runtime/glm53-spark-mtp3-mesh/image-receipt.json @@ -1,16 +1,36 @@ { - "added_layers": 6, + "added_layers": 8, "bundle_manifest_sha256": "4204fabc93303226b9a120b094ef3c82ed4aadd1d7f97cfbe291204c027ed45f", "checks_passed": true, - "image": "sparkring-glm53-spark-mtp3-mesh:managed-8684a696", - "image_id": "sha256:26273b8e358df139ae913610a5d43084ff0fd08aafe282ef633a3bc74afefe47", - "image_reference": "sha256:26273b8e358df139ae913610a5d43084ff0fd08aafe282ef633a3bc74afefe47", - "image_size_bytes": 21076522030, + "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache:glm53-spark-mtp3-nvfp4-a16-2a444f7c", + "image_id": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", + "image_reference": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", + "image_size_bytes": 22107698916, "inside_image": { - "b12x_commit": "9ae41c5cb9935d740456479954b0089f80bd2ef2", + "b12x_commit": "b58f34eaf978277621efced6678e6713fd7122e4", "bundle_files": 28, "bundle_manifest_sha256": "4204fabc93303226b9a120b094ef3c82ed4aadd1d7f97cfbe291204c027ed45f", "checks_passed": true, + "compute": { + "b12x_files": 385, + "b12x_revision": "b58f34eaf978277621efced6678e6713fd7122e4", + "b12x_tree": "7637fe5fb4d88882e0d18cdacc68c493f478499d", + "cuda_version": "13.3", + "environment": { + "CUDA_HOME": "/opt/cuda-13.3", + "TRITON_PTXAS_PATH": "/opt/cuda-13.3/bin/ptxas", + "VLLM_B12X_DENSE_ACTIVATION_MODE": "auto", + "VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH": "1", + "VLLM_LM_HEAD_A16": "1", + "VLLM_MTP_NVFP4_LM_HEAD": "1", + "VLLM_MXFP8_LM_HEAD": "0" + }, + "proposal_head_nvfp4": true, + "source_lock_sha256": "2a444f7c1ad4319f64afbffb16403491a4863934372cf818f18c510d58c0d00e", + "target_head_quantization": false, + "vllm_overrides": 14, + "vllm_parent_files": 2905 + }, "cuda_initialized": false, "device_access": false, "limitation": "Content and CPU checks do not qualify CUDA graphs, RDMA forwarding, native MTP, cache restoration, or model performance.", @@ -18,7 +38,7 @@ "marker_source_sha256": "8684a6961b8e86aa474fa2310ff71e4cdf219a63a72ceb5593b2f95e54812792", "model_loaded": false, "parent_package_files": { - "b12x": 374, + "b12x": 385, "sparkcache": 150, "vllm": 2905 }, @@ -26,22 +46,22 @@ "readiness_warmup": { "environment": "SPARKRING_WARMUP_TEMPERATURE", "helper_sha256": "f41c38eef41d15d63dcfc49cd6643357ca1a3ae18200ddbe4f8692d0b767ee79", - "temperature": 1 + "temperature": 1.0 }, "rocenante_lazy_import": "/opt/spark-sircl/b12x_overlay/b12x/comm/roce/__init__.py", "sircl_native_sha256": "61aa0ec56a1b438439bed8611dab0353d2c72c10af02bbd917fb77c87b33e5fc", - "source_receipt_sha256": "9578a15a76820c9253bc55df0a8d4c34d884cd540aeba5726796a0124062b361", + "source_receipt_sha256": "79dedb68ef65f51d99d66f8664e9235aa6fe8444205bb4c223e5046d076ab3c6", "sparkcache_commit": "66057174301a4759ca3a45207ea41016689449cb", "status": "research-only", "vllm_commit": "e02b174693e13859de61811b5e8cd13d5308e259", "vllm_native_extensions": 15 }, - "limitation": "This receipt covers device-free image-content verification. Four-rank serving and managed-fabric checks are recorded separately in performance/records/glm53-flash/spark-mtp3-managed-mesh-functional-20260905.md.", + "limitation": "No full-model, GPU, fabric, or four-rank serving test was performed for this image.", "parent_image_id": "sha256:5e32aaa1bbe3559e81db7706ed4286248f18d27cfdb186f6b851bf786eb43075", "parent_layers_retained": 81, "platform": "linux/arm64", "schema": "sparkring-mtp3-mesh-image-receipt/v1", - "source_receipt_sha256": "9578a15a76820c9253bc55df0a8d4c34d884cd540aeba5726796a0124062b361", + "source_receipt_sha256": "79dedb68ef65f51d99d66f8664e9235aa6fe8444205bb4c223e5046d076ab3c6", "status": "research-only", "verification_command": [ "docker", @@ -66,7 +86,7 @@ "PYTHONDONTWRITEBYTECODE=1", "--entrypoint", "python3", - "sha256:26273b8e358df139ae913610a5d43084ff0fd08aafe282ef633a3bc74afefe47", + "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", "-I", "/opt/sparkring/bin/verify-mtp3-mesh-image.py", "--inside-image" diff --git a/runtime/glm53-spark-mtp3-mesh/pins.json b/runtime/glm53-spark-mtp3-mesh/pins.json index 59a1b680..5cb80eac 100644 --- a/runtime/glm53-spark-mtp3-mesh/pins.json +++ b/runtime/glm53-spark-mtp3-mesh/pins.json @@ -2,6 +2,14 @@ "schema": "sparkring-glm53-spark-mtp3-mesh/v1", "status": "research-only", "image_pins": "../glm53-flash-jj-r8-gb10/pins.json", + "compute": { + "source_lock": "compute/source-lock.json", + "source_lock_sha256": "2a444f7c1ad4319f64afbffb16403491a4863934372cf818f18c510d58c0d00e", + "vllm_base_revision": "e02b174693e13859de61811b5e8cd13d5308e259", + "b12x_revision": "b58f34eaf978277621efced6678e6713fd7122e4", + "b12x_tree": "7637fe5fb4d88882e0d18cdacc68c493f478499d", + "cuda_version": "13.3" + }, "target": { "repository": "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark", "revision": "df116c4fb16b1d37ae43d2cfd624de26ffbc832e", @@ -28,7 +36,7 @@ "cache_identity": { "draft_policy": "separate", "draft_checkpoint_source": "target.checkpoint_identity", - "namespace": "glm53-spark-df116c4f-native-mtp3-mesh-4204fabc-tail-cow-v2", - "compatibility": "The native predictor is identified by its target checkpoint. This intentionally misses the measured deployment's external-draft-tagged entries. Cold-start and persistent-restore qualification of this namespace remain required." + "namespace": "glm53-spark-df116c4f-mtp3-nvfp4-a16-b58f34ea-mesh4204fabc-tail-cow-v2", + "compatibility": "The namespace separates the NVFP4/BF16 proposal head and B12X b58f34ea computation from cache entries produced with a shared BF16 head. The target checkpoint remains df116c4f; persistent restore requires this profile's matching compute and cache geometry." } } diff --git a/runtime/glm53-spark-mtp3-mesh/public-image.json b/runtime/glm53-spark-mtp3-mesh/public-image.json index 7ecf739a..0fe33ffb 100644 --- a/runtime/glm53-spark-mtp3-mesh/public-image.json +++ b/runtime/glm53-spark-mtp3-mesh/public-image.json @@ -1,18 +1,21 @@ { + "schema": "sparkring-managed-mesh-public-image/v1", + "status": "research-only", "checks_passed": true, "anonymous_manifest_read": true, "anonymous_config_read": true, "anonymous_pull": true, - "public_reference": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:23f00af873ccc784cfb742b7be2a29c6d3c20ebec9741843c025320bb9c04685", - "tag": "glm53-spark-mtp3-managed-8684a696", - "manifest_digest": "sha256:23f00af873ccc784cfb742b7be2a29c6d3c20ebec9741843c025320bb9c04685", - "config_image_id": "sha256:26273b8e358df139ae913610a5d43084ff0fd08aafe282ef633a3bc74afefe47", + "anonymous_pull_method": "Docker pull on the build host with an empty client credential directory; local image layers already present", + "public_reference": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86", + "tag": "glm53-spark-mtp3-nvfp4-a16-2a444f7c", + "manifest_digest": "sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86", + "config_image_id": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", "platform": "linux/arm64", - "layer_count": 87, + "layer_count": 89, "all_layer_diff_ids_match_tested_image": true, + "layer_identity_reference": "Built image verified by image-receipt.json; serving-image source parity is recorded separately", "package_url": "https://github.com/FujitsuPolycom/sparkring/pkgs/container/sparkring-glm53-sparkcache", - "schema": "sparkring-managed-mesh-public-image/v1", - "status": "research-only", "content_receipt": "image-receipt.json", - "functional_record": "../../performance/records/glm53-flash/spark-mtp3-managed-mesh-functional-20260905.md" + "compute_equivalence": "compute-image-equivalence.json", + "functional_record": "../../performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md" } diff --git a/runtime/glm53-spark-mtp3-mesh/test_image.py b/runtime/glm53-spark-mtp3-mesh/test_image.py index a7926873..0d2ba490 100644 --- a/runtime/glm53-spark-mtp3-mesh/test_image.py +++ b/runtime/glm53-spark-mtp3-mesh/test_image.py @@ -3,6 +3,7 @@ import importlib.util import json from pathlib import Path +from types import SimpleNamespace import pytest @@ -84,6 +85,239 @@ def test_file_map_rejects_parent_traversal(tmp_path): verifier.verify_file_map(tmp_path, {"../test": "0" * 64}) +def test_layered_file_map_checks_unchanged_parent_and_exact_overrides(tmp_path): + (tmp_path / "vllm").mkdir() + unchanged = tmp_path / "vllm" / "unchanged.py" + replaced = tmp_path / "vllm" / "replaced.py" + unchanged.write_text("parent\n", encoding="utf-8") + replaced.write_text("result\n", encoding="utf-8") + parent = { + "vllm/unchanged.py": verifier.sha256(unchanged), + "vllm/replaced.py": "0" * 64, + } + overrides = { + "vllm/replaced.py": { + "base_sha256": "0" * 64, + "result_sha256": verifier.sha256(replaced), + } + } + assert verifier.verify_layered_file_map(tmp_path, parent, overrides) == { + "parent_files": 2, + "overrides": 1, + } + unchanged.write_text("changed\n", encoding="utf-8") + with pytest.raises(ValueError, match="content pin"): + verifier.verify_layered_file_map(tmp_path, parent, overrides) + + +def test_layered_file_map_rejects_unbound_or_wrong_base_override(tmp_path): + (tmp_path / "vllm").mkdir() + source = tmp_path / "vllm" / "source.py" + source.write_text("result\n", encoding="utf-8") + parent = {"vllm/source.py": "1" * 64} + with pytest.raises(ValueError, match="base identity"): + verifier.verify_layered_file_map( + tmp_path, + parent, + {"vllm/source.py": { + "base_sha256": "2" * 64, + "result_sha256": verifier.sha256(source), + }}, + ) + with pytest.raises(ValueError, match="absent from the parent"): + verifier.verify_layered_file_map( + tmp_path, + parent, + {"vllm/other.py": { + "base_sha256": "1" * 64, + "result_sha256": verifier.sha256(source), + }}, + ) + + +def test_complete_package_file_map_rejects_stale_parent_module(tmp_path): + package = tmp_path / "b12x" + package.mkdir() + current = package / "current.py" + current.write_text("VALUE = 1\n", encoding="utf-8") + records = {"b12x/current.py": verifier.sha256(current)} + assert verifier.verify_complete_package_file_map(tmp_path, "b12x", records) == 1 + (package / "stale_parent.py").write_text("VALUE = 0\n", encoding="utf-8") + with pytest.raises(ValueError, match="complete manifest"): + verifier.verify_complete_package_file_map(tmp_path, "b12x", records) + + +def test_compute_environment_requires_cuda_and_quantization_contract(): + expected = { + "CUDA_HOME": "/opt/cuda-13.3", + "TRITON_PTXAS_PATH": "/opt/cuda-13.3/bin/ptxas", + "VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH": "1", + "VLLM_B12X_DENSE_ACTIVATION_MODE": "auto", + "VLLM_MTP_NVFP4_LM_HEAD": "1", + "VLLM_LM_HEAD_A16": "1", + "VLLM_MXFP8_LM_HEAD": "0", + } + assert verifier.verify_required_environment(expected, expected) == expected + for name in expected: + changed = dict(expected) + changed[name] = "wrong" + with pytest.raises(ValueError, match=name): + verifier.verify_required_environment(changed, expected) + + +def test_compute_verifier_composes_parent_vllm_and_complete_b12x(tmp_path): + compute = tmp_path / "compute" + receipts = tmp_path / "receipts" + site = tmp_path / "site" + cuda = tmp_path / "cuda" + for path in (compute, receipts, site / "vllm", site / "b12x", cuda / "bin"): + path.mkdir(parents=True, exist_ok=True) + unchanged = site / "vllm" / "unchanged.py" + override = site / "vllm" / "override.py" + b12x_python = site / "b12x" / "__init__.py" + b12x_notice = site / "b12x" / "README.md" + unchanged.write_text("unchanged\n", encoding="utf-8") + override.write_text("result\n", encoding="utf-8") + b12x_python.write_text("", encoding="utf-8") + b12x_notice.write_text("source\n", encoding="utf-8") + (cuda / "bin" / "ptxas").write_text("tool\n", encoding="utf-8") + components = {"cuda_nvcc/archive.tar.xz": "a" * 64} + (cuda / "sparkring-component-manifest.json").write_text( + json.dumps(components), encoding="utf-8" + ) + base_hash = "b" * 64 + lock = { + "schema": "sparkring-glm53-compute-source/v1", + "vllm": { + "base_revision": "e02", + "files": [["vllm/override.py", base_hash, verifier.sha256(override)]], + }, + "b12x": { + "revision": "b58", + "tree": "tree", + "package_files_sha256": verifier.file_map_sha256({ + "b12x/__init__.py": verifier.sha256(b12x_python), + "b12x/README.md": verifier.sha256(b12x_notice), + }), + }, + "cuda": {"version": "13.3", "components": components}, + "environment": { + "VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH": "1", + "VLLM_B12X_DENSE_ACTIVATION_MODE": "auto", + "VLLM_MTP_NVFP4_LM_HEAD": "1", + "VLLM_LM_HEAD_A16": "1", + "VLLM_MXFP8_LM_HEAD": "0", + }, + } + lock_path = compute / "source-lock.json" + lock_path.write_text(json.dumps(lock), encoding="utf-8") + lock_hash = verifier.sha256(lock_path) + (receipts / "vllm-source-manifest.json").write_text( + json.dumps({ + "commit": "e02", + "files": { + "vllm/unchanged.py": verifier.sha256(unchanged), + "vllm/override.py": base_hash, + }, + }), + encoding="utf-8", + ) + b12x_files = { + "b12x/__init__.py": verifier.sha256(b12x_python), + "b12x/README.md": verifier.sha256(b12x_notice), + } + installed_path = tmp_path / "installed.json" + installed_path.write_text( + json.dumps({ + "schema": "sparkring-glm53-compute-installed/v1", + "source_lock_sha256": lock_hash, + "vllm_revision": "e02", + "vllm_overrides": {"vllm/override.py": verifier.sha256(override)}, + "b12x_revision": "b58", + "b12x_tree": "tree", + "b12x_files": b12x_files, + "cuda_components": components, + "environment": lock["environment"], + "target_head_quantization": False, + }), + encoding="utf-8", + ) + profile = {"compute": { + "source_lock": "compute/source-lock.json", + "source_lock_sha256": lock_hash, + "vllm_base_revision": "e02", + "b12x_revision": "b58", + "b12x_tree": "tree", + "cuda_version": "13.3", + }} + source = {"files": { + "compute/source-lock.json": lock_hash, + "compute/b12x-source/b12x/__init__.py": verifier.sha256(b12x_python), + "compute/b12x-source/b12x/README.md": verifier.sha256(b12x_notice), + "compute/b12x-source/pyproject.toml": "a" * 64, + "compute/b12x-source/tests/test_example.py": "b" * 64, + }} + environment = { + **lock["environment"], + "CUDA_HOME": "/opt/cuda-13.3", + "TRITON_PTXAS_PATH": "/opt/cuda-13.3/bin/ptxas", + } + result = verifier.verify_compute( + profile, + {"vllm": {"commit": "e02"}}, + source, + environment, + compute_root=compute, + receipt_path=installed_path, + site=site, + base_receipts=receipts, + cuda_root_override=cuda, + ptxas_runner=lambda *args, **kwargs: SimpleNamespace( + stdout="ptxas release 13.3", stderr="" + ), + ) + assert result["vllm_parent_files"] == 2 + assert result["vllm_overrides"] == 1 + assert result["b12x_files"] == 2 + assert result["proposal_head_nvfp4"] is True + assert result["target_head_quantization"] is False + + changed_source = json.loads(json.dumps(source)) + changed_source["files"]["compute/b12x-source/b12x/README.md"] = "f" * 64 + with pytest.raises(ValueError, match="source-receipt-bound"): + verifier.verify_compute( + profile, + {"vllm": {"commit": "e02"}}, + changed_source, + environment, + compute_root=compute, + receipt_path=installed_path, + site=site, + base_receipts=receipts, + cuda_root_override=cuda, + ptxas_runner=lambda *args, **kwargs: SimpleNamespace( + stdout="ptxas release 13.3", stderr="" + ), + ) + + unchanged.write_text("changed\n", encoding="utf-8") + with pytest.raises(ValueError, match="content pin"): + verifier.verify_compute( + profile, + {"vllm": {"commit": "e02"}}, + source, + environment, + compute_root=compute, + receipt_path=installed_path, + site=site, + base_receipts=receipts, + cuda_root_override=cuda, + ptxas_runner=lambda *args, **kwargs: SimpleNamespace( + stdout="ptxas release 13.3", stderr="" + ), + ) + + def test_container_verification_has_no_device_or_network_access(): source = (HERE / "verify_mesh_image.py").read_text(encoding="utf-8") assert '"--network", "none"' in source @@ -111,6 +345,27 @@ def test_pins_use_native_mtp_only(): assert pins["target"]["repository"].endswith("-Spark") +def test_profile_pins_exact_compute_source_and_quantization_environment(): + pins = json.loads((HERE / "pins.json").read_text(encoding="utf-8")) + compute = pins["compute"] + lock_path = HERE / compute["source_lock"] + lock = json.loads(lock_path.read_text(encoding="utf-8")) + assert verifier.sha256(lock_path) == compute["source_lock_sha256"] + assert lock["schema"] == "sparkring-glm53-compute-source/v1" + assert lock["vllm"]["base_revision"] == compute["vllm_base_revision"] + assert lock["b12x"]["revision"] == compute["b12x_revision"] + assert lock["b12x"]["tree"] == compute["b12x_tree"] + assert lock["cuda"]["version"] == compute["cuda_version"] == "13.3" + assert lock["environment"] == { + "VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH": "1", + "VLLM_B12X_DENSE_ACTIVATION_MODE": "auto", + "VLLM_MTP_NVFP4_LM_HEAD": "1", + "VLLM_LM_HEAD_A16": "1", + "VLLM_MXFP8_LM_HEAD": "0", + } + assert len(lock["vllm"]["files"]) == 14 + + def test_schema_accepts_research_only_status(): schema = json.loads((HERE / "image-receipt.schema.json").read_text(encoding="utf-8")) assert schema["properties"]["status"]["const"] == "research-only" diff --git a/runtime/glm53-spark-mtp3-mesh/verify_mesh_image.py b/runtime/glm53-spark-mtp3-mesh/verify_mesh_image.py index 84bd2608..4d22cb27 100644 --- a/runtime/glm53-spark-mtp3-mesh/verify_mesh_image.py +++ b/runtime/glm53-spark-mtp3-mesh/verify_mesh_image.py @@ -18,6 +18,8 @@ BASE_RECEIPTS = Path("/opt/sparkring/receipts/jj-r8-sparkcache-arm64") SITE = Path("/usr/local/lib/python3.12/dist-packages") BUNDLE = Path("/opt/spark-sircl") +COMPUTE = Path("/opt/sparkring-compute") +COMPUTE_RECEIPT = Path("/opt/sparkring/receipts/glm53-compute-installed.json") def sha256(path: Path) -> str: @@ -28,6 +30,11 @@ def sha256(path: Path) -> str: return digest.hexdigest() +def file_map_sha256(records: dict) -> str: + payload = json.dumps(records, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(payload).hexdigest() + + def load(path: Path) -> dict: value = json.loads(path.read_text(encoding="utf-8")) if not isinstance(value, dict): @@ -51,6 +58,181 @@ def verify_file_map(root: Path, records: dict) -> int: return len(records) +def verify_layered_file_map(root: Path, parent_records: dict, + overrides: dict) -> dict: + """Verify every parent file, substituting only manifest-bound overrides.""" + if not isinstance(parent_records, dict) or not parent_records: + raise ValueError("Parent source file manifest is empty") + if not isinstance(overrides, dict) or not overrides: + raise ValueError("Compute override manifest is empty") + expected = dict(parent_records) + for relative, record in overrides.items(): + if relative not in parent_records: + raise ValueError(f"Compute override is absent from the parent manifest: {relative}") + if not isinstance(record, dict): + raise ValueError(f"Compute override record is invalid: {relative}") + if record.get("base_sha256") != parent_records[relative]: + raise ValueError(f"Compute override base identity differs from the parent: {relative}") + result = record.get("result_sha256") + if not isinstance(result, str) or len(result) != 64: + raise ValueError(f"Compute override result identity is invalid: {relative}") + expected[relative] = result + verify_file_map(root, expected) + return {"parent_files": len(parent_records), "overrides": len(overrides)} + + +def verify_complete_package_file_map(root: Path, package: str, + records: dict) -> int: + """Verify all installed package files and reject inherited stale files.""" + prefix = f"{package}/" + if not isinstance(records, dict) or not records: + raise ValueError(f"{package} source manifest is empty") + if any(not relative.startswith(prefix) for relative in records): + raise ValueError(f"{package} source manifest contains an invalid path") + verify_file_map(root, records) + observed = { + path.relative_to(root).as_posix() + for path in (root / package).rglob("*") + if (path.is_file() and not path.is_symlink() + and "__pycache__" not in path.parts and path.suffix != ".pyc") + } + if observed != set(records): + missing = sorted(set(records) - observed) + extra = sorted(observed - set(records)) + raise ValueError( + f"{package} source set differs from its complete manifest; " + f"missing={missing}, extra={extra}" + ) + return len(records) + + +def verify_required_environment(environment: dict, expected: dict) -> dict: + """Require exact construction-time values for compute-selection settings.""" + if not isinstance(expected, dict) or not expected: + raise ValueError("Required compute environment is empty") + for name, value in expected.items(): + if environment.get(name) != value: + raise ValueError( + f"Mesh compute environment differs from its required value: {name}" + ) + return dict(sorted(expected.items())) + + +def verify_compute(profile: dict, base: dict, source: dict, environment: dict, + *, compute_root: Path = COMPUTE, + receipt_path: Path = COMPUTE_RECEIPT, + site: Path = SITE, + base_receipts: Path = BASE_RECEIPTS, + cuda_root_override: Path | None = None, + ptxas_runner=subprocess.run) -> dict: + """Verify the manifest-bound vLLM, B12X, and CUDA compute composition.""" + pin = profile.get("compute") + required_pin_fields = { + "source_lock", "source_lock_sha256", "vllm_base_revision", + "b12x_revision", "b12x_tree", "cuda_version", + } + if not isinstance(pin, dict) or not required_pin_fields.issubset(pin): + raise ValueError("Mesh profile does not bind the required compute source") + if pin["source_lock"] != "compute/source-lock.json": + raise ValueError("Mesh profile compute source-lock locator is unsupported") + lock_path = compute_root / "source-lock.json" + check_file(lock_path, pin["source_lock_sha256"]) + if source["files"].get("compute/source-lock.json") != pin["source_lock_sha256"]: + raise ValueError("Image source receipt does not bind the compute source lock") + lock = load(lock_path) + if lock.get("schema") != "sparkring-glm53-compute-source/v1": + raise ValueError("Compute source lock uses an unsupported schema") + if lock["vllm"]["base_revision"] != pin["vllm_base_revision"]: + raise ValueError("Compute vLLM base revision differs from the profile pin") + if lock["b12x"]["revision"] != pin["b12x_revision"]: + raise ValueError("Compute B12X revision differs from the profile pin") + if lock["b12x"]["tree"] != pin["b12x_tree"]: + raise ValueError("Compute B12X tree differs from the profile pin") + if lock["cuda"]["version"] != pin["cuda_version"]: + raise ValueError("Compute CUDA version differs from the profile pin") + + installed = load(receipt_path) + if installed.get("schema") != "sparkring-glm53-compute-installed/v1": + raise ValueError("Installed compute receipt uses an unsupported schema") + if installed.get("source_lock_sha256") != pin["source_lock_sha256"]: + raise ValueError("Installed compute receipt uses a different source lock") + if installed.get("vllm_revision") != lock["vllm"]["base_revision"]: + raise ValueError("Installed vLLM base revision differs from the source lock") + if (installed.get("b12x_revision") != lock["b12x"]["revision"] + or installed.get("b12x_tree") != lock["b12x"]["tree"]): + raise ValueError("Installed B12X identity differs from the source lock") + + parent_vllm = load(base_receipts / "vllm-source-manifest.json") + if parent_vllm.get("commit") != base["vllm"]["commit"]: + raise ValueError("Parent source receipt has the wrong vllm revision") + if base["vllm"]["commit"] != lock["vllm"]["base_revision"]: + raise ValueError("Compute source does not extend the pinned parent vLLM") + overrides = { + relative: {"base_sha256": parent_hash, "result_sha256": result_hash} + for relative, parent_hash, result_hash in lock["vllm"]["files"] + } + expected_results = { + relative: record["result_sha256"] for relative, record in overrides.items() + } + if installed.get("vllm_overrides") != expected_results: + raise ValueError("Installed vLLM override map differs from the source lock") + vllm = verify_layered_file_map(site, parent_vllm["files"], overrides) + + b12x_prefix = "compute/b12x-source/" + expected_b12x_files = { + relative.removeprefix(b12x_prefix): expected + for relative, expected in source["files"].items() + if relative.startswith(b12x_prefix + "b12x/") + } + b12x_files = installed.get("b12x_files") + if b12x_files != expected_b12x_files: + raise ValueError( + "Installed B12X file map differs from the source-receipt-bound source" + ) + if file_map_sha256(b12x_files) != lock["b12x"]["package_files_sha256"]: + raise ValueError("Installed B12X file-map identity differs from the source lock") + b12x_count = verify_complete_package_file_map(site, "b12x", b12x_files) + if installed.get("cuda_components") != lock["cuda"]["components"]: + raise ValueError("Installed CUDA component map differs from the source lock") + cuda_root = cuda_root_override or Path(f"/opt/cuda-{lock['cuda']['version']}") + cuda_manifest = load(cuda_root / "sparkring-component-manifest.json") + if cuda_manifest != lock["cuda"]["components"]: + raise ValueError("CUDA component manifest differs from the source lock") + ptxas = cuda_root / "bin/ptxas" + if ptxas.is_symlink() or not ptxas.is_file(): + raise ValueError("Pinned CUDA toolkit does not contain ptxas") + version_result = ptxas_runner( + [str(ptxas), "--version"], capture_output=True, text=True, check=True + ) + version_text = version_result.stdout + version_result.stderr + if f"release {lock['cuda']['version']}" not in version_text: + raise ValueError("Installed ptxas version differs from the CUDA source lock") + expected_environment = { + **lock["environment"], + "CUDA_HOME": f"/opt/cuda-{lock['cuda']['version']}", + "TRITON_PTXAS_PATH": f"/opt/cuda-{lock['cuda']['version']}/bin/ptxas", + } + if installed.get("environment") != lock["environment"]: + raise ValueError("Installed compute environment differs from the source lock") + verified_environment = verify_required_environment( + environment, expected_environment + ) + if installed.get("target_head_quantization") is not False: + raise ValueError("Target LM head must remain unquantized") + return { + "source_lock_sha256": pin["source_lock_sha256"], + "vllm_parent_files": vllm["parent_files"], + "vllm_overrides": vllm["overrides"], + "b12x_revision": lock["b12x"]["revision"], + "b12x_tree": lock["b12x"]["tree"], + "b12x_files": b12x_count, + "cuda_version": lock["cuda"]["version"], + "environment": verified_environment, + "proposal_head_nvfp4": True, + "target_head_quantization": False, + } + + def verify_warmup(path: Path, expected: str, environment: dict) -> dict: """Check the readiness-only helper override and its explicit temperature.""" check_file(path, expected) @@ -83,12 +265,15 @@ def verify_inside_image() -> dict: check_file(marker_source, profile["marker"]["source_sha256"]) warmup = verify_warmup(Path("/opt/sparkring/bin/warmup_dflash.py"), source["files"]["warmup_dflash.py"], os.environ) - package_counts = {} - for package in ("vllm", "b12x", "sparkcache"): - manifest = load(BASE_RECEIPTS / f"{package}-source-manifest.json") - if manifest.get("commit") != base[package]["commit"]: - raise ValueError(f"Parent source receipt has the wrong {package} revision") - package_counts[package] = verify_file_map(SITE, manifest["files"]) + compute = verify_compute(profile, base, source, os.environ) + sparkcache_manifest = load(BASE_RECEIPTS / "sparkcache-source-manifest.json") + if sparkcache_manifest.get("commit") != base["sparkcache"]["commit"]: + raise ValueError("Parent source receipt has the wrong sparkcache revision") + package_counts = { + "vllm": compute["vllm_parent_files"], + "b12x": compute["b12x_files"], + "sparkcache": verify_file_map(SITE, sparkcache_manifest["files"]), + } native = load(BASE_RECEIPTS / "native-extension-manifest.json")["files"] native_count = verify_file_map(SITE / "vllm", native) check_file(BUNDLE / "libspark_transport_capi.so", base["sircl"]["native_sha256"]) @@ -117,12 +302,14 @@ def verify_inside_image() -> dict: "source_receipt_sha256": sha256(RECEIPTS / "source-receipt.json"), "bundle_files": verified_bundle, "python_syntax_files": python_files, "parent_package_files": package_counts, "vllm_native_extensions": native_count, - "vllm_commit": base["vllm"]["commit"], "b12x_commit": base["b12x"]["commit"], + "vllm_commit": base["vllm"]["commit"], + "b12x_commit": compute["b12x_revision"], "sparkcache_commit": base["sparkcache"]["commit"], "sircl_native_sha256": base["sircl"]["native_sha256"], "marker_source_sha256": sha256(marker_source), "marker_binary_sha256": sha256(marker), "rocenante_lazy_import": str(roce.__file__), "cuda_initialized": False, "device_access": False, "model_loaded": False, + "compute": compute, "readiness_warmup": warmup, "limitation": "Content and CPU checks do not qualify CUDA graphs, RDMA forwarding, native MTP, cache restoration, or model performance.", } From a86aa58a75c212502f0b72d447be51db29b89393 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:41:16 -0500 Subject: [PATCH 03/16] Order RoCEnante staging and enforce mesh compute receipts Preserve per-capture stream admission, sequence shared staging and output copies, and reject images without the pinned compute configuration. Record per-run benchmark variability and bundled vLLM provenance. Validation: 372 CPU tests passed, 21 skipped; ARM64 image content checks and anonymous publication passed. GPU stream/fault validation remains separate from these CPU checks. --- THIRD_PARTY_NOTICES.md | 15 +- docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md | 9 +- ...spark-mtp3-nvfp4-proposal-head-20260905.md | 23 + ...-mtp3-nvfp4-proposal-samples-20260905.json | 1272 +++++++++++++++++ .../glm53-spark-mtp3-managed-mesh-tp4.json | 6 +- runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md | 6 +- .../compute-image-equivalence.json | 4 +- .../glm53-spark-mtp3-mesh/image-receipt.json | 18 +- runtime/glm53-spark-mtp3-mesh/pins.json | 4 +- runtime/glm53-spark-mtp3-mesh/profile.py | 15 + .../glm53-spark-mtp3-mesh/public-image.json | 8 +- .../qualification/test_checks.py | 3 +- runtime/glm53-spark-mtp3-mesh/test_profile.py | 16 +- .../rocenante_vllm_overlay.py | 3 +- .../test_stream_lifetime.py | 81 ++ third_party/b12x_roce/README.md | 4 + .../b12x_roce/b12x/comm/roce/roce_oneshot.py | 50 +- third_party/b12x_roce/provenance.json | 3 +- 18 files changed, 1482 insertions(+), 58 deletions(-) create mode 100644 performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-samples-20260905.json create mode 100644 spark_transport/experiments/glm53_rocenante_overlay/test_stream_lifetime.py diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index c3d03400..71f50c51 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -55,7 +55,14 @@ two compatibility patches. The public GLM-5.3 builder does not consume them; its `NCCL_SWITCHLESS_RING_ONLY` parameter and diagnostics are an independent SparkRing implementation. -## 3. vLLM (referenced and patched) +## 3. vLLM (referenced, patched, and selected source included) + +`runtime/glm53-spark-mtp3-mesh/compute/vllm-compute-files.tar.gz` includes +fourteen vLLM Python files for GLM metadata reuse, dense-kernel integration, +and the NVFP4 proposal head. The adjacent patch provides a readable diff; +`source-lock.json` records the base, donor revisions, and exact file hashes. +These files derive from Local Inference Lab's vLLM fork at `3512b066` and +`a8c796f3`, under Apache-2.0 with their contributor notices retained. The unified diffs under `runtime/deepseek0731-gb10/patches/` contain context and removed lines from vLLM, pinned to the source revision recorded by that @@ -245,8 +252,10 @@ artifacts that are not distributed in this repository: repository notices. SparkRing records these identities and validates compatible image content; it -does not redistribute the model weights, vLLM source, or B12X model-kernel -package. The selected B12X communication source in Section 11 is included. Operators +does not include model weights or the complete vLLM/B12X source trees in this +Git repository. The selected vLLM files in Section 3 and B12X communication +source in Section 11 are included. Published runtime images contain the +pinned vLLM and B12X packages under their respective licenses. Operators must obtain each artifact under its own terms. The exact operator-image composition is in `runtime/glm53-flash-jj-r8-gb10/pins.json` and `runtime/glm53-flash-jj-r8-gb10/glm53-dcp4-sircl-public-image-receipt.json`. diff --git a/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md b/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md index bb5c9e89..cc09f649 100644 --- a/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md +++ b/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md @@ -35,6 +35,9 @@ DFlash model is used. The [profile contract](../runtime/glm53-spark-mtp3-mesh/README.md) and [pins](../runtime/glm53-spark-mtp3-mesh/pins.json) are the canonical inputs. +The packaged RoCEnante runtime orders shared staging buffers across streams +and preserves its one-stream-per-CUDA-capture guard. These fixes have CPU +regression coverage; four-rank GPU fault and stream tests remain required. The [proposal-head throughput record](../performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md) reports observations, not a general performance guarantee. @@ -391,8 +394,8 @@ That proves build/content equivalence, not a fresh serving, restart, or persistent-cache qualification for the public image ID. ```bash -mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86' -mtp_image_id='sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df' +mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:b65d427f9be49c97d57e404ad1a6118769c1119df876a8944c1f186a6b380c5d' +mtp_image_id='sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766' docker pull "${mtp_image}" test "$(docker image inspect "${mtp_image}" --format '{{.Id}}')" = "${mtp_image_id}" @@ -443,7 +446,7 @@ docker rm sparkring-mtp3-extract cp runtime/glm53-spark-mtp3-mesh/image-receipt.json /srv/sparkring/verified-image-receipt.json printf '%s %s\n' \ - '4204fabc93303226b9a120b094ef3c82ed4aadd1d7f97cfbe291204c027ed45f' \ + '69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500' \ '/srv/sparkring/artifacts/mtp3-mesh-bundle/sparkring-overlay-manifest.json' \ '2828c07e4255c4962c77425be2c88969e7eb7dd4b1bf9e36485bc705bb5d6d64' \ '/srv/sparkring/artifacts/mlx5-rdma-tx-marker' | sha256sum --check diff --git a/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md b/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md index f4a3d465..5d633d34 100644 --- a/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md +++ b/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md @@ -58,6 +58,29 @@ restart, and persistent-cache checks were not repeated on the public image ID. ## Measurement +[Sanitized per-run samples](spark-mtp3-nvfp4-proposal-samples-20260905.json) +include all decode windows, token counts, per-context prefill scouts, and +recorded settings for the two control and three proposal repetitions. +Each decode cell requests five seconds of warmup, allows 900 seconds to reach +the requested concurrency, and measures a 20-second client wall-clock window. +The harness version is 0.4.32; its exact historical source commit was not +captured. The recorded equivalent command is: + +```bash +python llm_decode_bench.py --host http://RANK0 --port 8015 \ + --model glm-5.3-flash-spark --temperature 1.0 --token-targeting exact \ + --display-mode live --no-hw-monitor --dcp-size 4 \ + --concurrency 1,2,4,8 --contexts 8k,16k --max-tokens 2048 \ + --duration 20 --decode-warmup-seconds 5 \ + --cell-warmup-timeout-seconds 900 --output result.json +``` + +Variability is reported as the minimum and maximum across repetitions, not +as a confidence interval. At 8K/C1, control output throughput spans +46.61–48.84 tok/s and proposal throughput spans 49.35–52.80 tok/s; +normalized throughput spans 17.94–18.01 and 18.70–19.05 respectively. +The sample file contains the same ranges for every measured 8K concurrency. + Raw throughput is aggregate output tokens per second. Normalized throughput is the harness's aggregate sequence steps per second, computed from drafted and non-speculative request work. It is not a count of batched engine iterations. diff --git a/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-samples-20260905.json b/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-samples-20260905.json new file mode 100644 index 00000000..ac9ca257 --- /dev/null +++ b/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-samples-20260905.json @@ -0,0 +1,1272 @@ +{ + "schema": "sparkring-sanitized-proposal-benchmark-samples/v1", + "harness_revision": "Recorded version 0.4.32; exact historical source commit was not captured", + "timing": "Client wall-clock measurement window; aggregate output uses OpenAI continuous-usage token counts", + "variability": "Minimum and maximum across repetitions at each 8K concurrency; not confidence intervals", + "ranges": { + "control": [ + { + "concurrency": 1, + "aggregate_tps_min": 46.61270983199115, + "aggregate_tps_max": 48.839071256871314, + "normalized_min": 17.93565147876187, + "normalized_max": 18.014411529173845 + }, + { + "concurrency": 2, + "aggregate_tps_min": 74.55964771796953, + "aggregate_tps_max": 77.1430004517391, + "normalized_min": 28.122497998321393, + "normalized_max": 28.93488271324153 + }, + { + "concurrency": 4, + "aggregate_tps_min": 118.67889540432174, + "aggregate_tps_max": 120.7303004461975, + "normalized_min": 43.30175913400928, + "normalized_max": 43.336510006445636 + }, + { + "concurrency": 8, + "aggregate_tps_min": 163.50687130131578, + "aggregate_tps_max": 171.3898781157694, + "normalized_min": 59.384090681214076, + "normalized_max": 60.189597231174496 + } + ], + "proposal": [ + { + "concurrency": 1, + "aggregate_tps_min": 49.35, + "aggregate_tps_max": 52.8, + "normalized_min": 18.7, + "normalized_max": 19.05 + }, + { + "concurrency": 2, + "aggregate_tps_min": 74.65599199404768, + "aggregate_tps_max": 78.9, + "normalized_min": 28.22116587442553, + "normalized_max": 29.600000000000005 + }, + { + "concurrency": 4, + "aggregate_tps_min": 115.25, + "aggregate_tps_max": 123.99137974179853, + "normalized_min": 41.8, + "normalized_max": 44.10364356241686 + }, + { + "concurrency": 8, + "aggregate_tps_min": 164.26744244341373, + "aggregate_tps_max": 173.45, + "normalized_min": 60.89743589745578, + "normalized_max": 61.39338917579799 + } + ] + }, + "runs": [ + { + "id": "114306", + "source_sha256": "f7ae32cef4c9ea606477a28ff0465403e78f9519e2d771e4af0f13017b671677", + "metadata": { + "version": "0.4.32", + "engine": "vllm", + "model": "glm-5.3-flash-spark", + "decode_mode": "duration", + "duration_per_test": 20, + "decode_warmup_seconds": 5, + "decode_warmup_context": 16384, + "decode_warmup_concurrency": 1, + "cell_warmup_timeout_seconds": 900, + "max_tokens": 2048, + "temperature": 1, + "ignore_eos": true, + "dcp_size": 4, + "concurrency_levels": [ + 1, + 2, + 4, + 8 + ], + "context_lengths": [ + 8192, + 16384 + ], + "prefill_mode": "integrated_decode_scout" + }, + "prefill": { + "8192": { + "ttft_seconds": 3.078, + "prefill_seconds": 3.078, + "tok_per_sec": 2661, + "client_ttft_seconds": 3.078, + "client_tok_per_sec": 2661, + "prompt_tokens": 8192, + "samples": 1, + "method": "integrated_scout", + "server_validation": { + "method": "prometheus:kv_computed", + "tok_per_sec": 2688, + "prefill_seconds": 3.047, + "prompt_tokens": 8192, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 1, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "16384": { + "ttft_seconds": 6.047, + "prefill_seconds": 6.047, + "tok_per_sec": 2709, + "client_ttft_seconds": 6.047, + "client_tok_per_sec": 2709, + "prompt_tokens": 16384, + "samples": 1, + "method": "integrated_scout", + "server_validation": { + "method": "prometheus:kv_computed", + "tok_per_sec": 2722, + "prefill_seconds": 6.019, + "prompt_tokens": 16384, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 1, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "65536": { + "ttft_seconds": 23.578, + "prefill_seconds": 23.578, + "tok_per_sec": 2780, + "client_ttft_seconds": 23.578, + "client_tok_per_sec": 2780, + "prompt_tokens": 65536, + "samples": 1, + "method": "scout_only", + "server_validation": { + "method": "", + "tok_per_sec": 0, + "prefill_seconds": 0, + "prompt_tokens": 0, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 0, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "131072": { + "ttft_seconds": 47.531, + "prefill_seconds": 47.531, + "tok_per_sec": 2758, + "client_ttft_seconds": 47.531, + "client_tok_per_sec": 2758, + "prompt_tokens": 131072, + "samples": 1, + "method": "scout_only", + "server_validation": { + "method": "", + "tok_per_sec": 0, + "prefill_seconds": 0, + "prompt_tokens": 0, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 0, + "invalid_reason": "" + }, + "hardware_summary": {} + } + }, + "decode": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 48.839071256871314, + "server_steps_per_s": 18.014411529173845, + "server_spec_accept_length": 2.7111111111111112, + "measurement_wall_seconds": 20.015, + "client_output_tokens": 976, + "server_output_tokens": 976, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 12.907 + }, + { + "context_tokens": 16384, + "concurrency": 1, + "aggregate_tps": 49.93745308985227, + "server_steps_per_s": 17.813360020027464, + "server_spec_accept_length": 2.803370786516854, + "measurement_wall_seconds": 20, + "client_output_tokens": 998, + "server_output_tokens": 998, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 13.453 + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 74.55964771796953, + "server_steps_per_s": 28.122497998321393, + "server_spec_accept_length": 2.6512455516014235, + "measurement_wall_seconds": 20.015, + "client_output_tokens": 1490, + "server_output_tokens": 1490, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 6.625 + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 120.7303004461975, + "server_steps_per_s": 43.336510006445636, + "server_spec_accept_length": 2.7858796296296298, + "measurement_wall_seconds": 20, + "client_output_tokens": 2407, + "server_output_tokens": 2407, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 8.704 + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 163.50687130131578, + "server_steps_per_s": 59.384090681214076, + "server_spec_accept_length": 2.753378378378378, + "measurement_wall_seconds": 20, + "client_output_tokens": 3260, + "server_output_tokens": 3260, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 12.234 + }, + { + "context_tokens": 16384, + "concurrency": 2, + "aggregate_tps": 75.0062546910707, + "server_steps_per_s": 28.42131598701011, + "server_spec_accept_length": 2.6390845070422535, + "measurement_wall_seconds": 20.016, + "client_output_tokens": 1499, + "server_output_tokens": 1499, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 7.156 + }, + { + "context_tokens": 16384, + "concurrency": 4, + "aggregate_tps": 113.46664661967247, + "server_steps_per_s": 41.71421133423315, + "server_spec_accept_length": 2.7200956937799043, + "measurement_wall_seconds": 20, + "client_output_tokens": 2264, + "server_output_tokens": 2274, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 9.187 + }, + { + "context_tokens": 16384, + "concurrency": 8, + "aggregate_tps": 163.28993011960046, + "server_steps_per_s": 59.524408023893756, + "server_spec_accept_length": 2.743243243243243, + "measurement_wall_seconds": 20.016, + "client_output_tokens": 3248, + "server_output_tokens": 3248, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 13.297 + } + ] + }, + { + "id": "115629", + "source_sha256": "7c3ff3210e805b8a801d4e8ea49152d0545a982707cc9010a174f77ff3d4e673", + "metadata": { + "version": "0.4.32", + "engine": "vllm", + "model": "glm-5.3-flash-spark", + "decode_mode": "duration", + "duration_per_test": 20, + "decode_warmup_seconds": 5, + "decode_warmup_context": 16384, + "decode_warmup_concurrency": 1, + "cell_warmup_timeout_seconds": 900, + "max_tokens": 2048, + "temperature": 1, + "ignore_eos": true, + "dcp_size": 4, + "concurrency_levels": [ + 1, + 2, + 4, + 8 + ], + "context_lengths": [ + 8192, + 16384 + ], + "prefill_mode": "integrated_decode_scout" + }, + "prefill": { + "8192": { + "ttft_seconds": 3.063, + "prefill_seconds": 3.063, + "tok_per_sec": 2675, + "client_ttft_seconds": 3.063, + "client_tok_per_sec": 2675, + "prompt_tokens": 8192, + "samples": 1, + "method": "integrated_scout", + "server_validation": { + "method": "prometheus:kv_computed", + "tok_per_sec": 2693, + "prefill_seconds": 3.042, + "prompt_tokens": 8192, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 1, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "16384": { + "ttft_seconds": 6.047, + "prefill_seconds": 6.047, + "tok_per_sec": 2709, + "client_ttft_seconds": 6.047, + "client_tok_per_sec": 2709, + "prompt_tokens": 16384, + "samples": 1, + "method": "integrated_scout", + "server_validation": { + "method": "prometheus:kv_computed", + "tok_per_sec": 2724, + "prefill_seconds": 6.014, + "prompt_tokens": 16384, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 1, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "65536": { + "ttft_seconds": 23.532, + "prefill_seconds": 23.532, + "tok_per_sec": 2785, + "client_ttft_seconds": 23.532, + "client_tok_per_sec": 2785, + "prompt_tokens": 65536, + "samples": 1, + "method": "scout_only", + "server_validation": { + "method": "", + "tok_per_sec": 0, + "prefill_seconds": 0, + "prompt_tokens": 0, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 0, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "131072": { + "ttft_seconds": 47.828, + "prefill_seconds": 47.828, + "tok_per_sec": 2740, + "client_ttft_seconds": 47.828, + "client_tok_per_sec": 2740, + "prompt_tokens": 131072, + "samples": 1, + "method": "scout_only", + "server_validation": { + "method": "", + "tok_per_sec": 0, + "prefill_seconds": 0, + "prompt_tokens": 0, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 0, + "invalid_reason": "" + }, + "hardware_summary": {} + } + }, + "decode": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 46.61270983199115, + "server_steps_per_s": 17.93565147876187, + "server_spec_accept_length": 2.598885793871866, + "measurement_wall_seconds": 20.016, + "client_output_tokens": 933, + "server_output_tokens": 933, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 12.437 + }, + { + "context_tokens": 16384, + "concurrency": 1, + "aggregate_tps": 47.774049777056554, + "server_steps_per_s": 17.92150874604316, + "server_spec_accept_length": 2.6657381615598883, + "measurement_wall_seconds": 20.016, + "client_output_tokens": 954, + "server_output_tokens": 957, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 13.937 + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 77.1430004517391, + "server_steps_per_s": 28.93488271324153, + "server_spec_accept_length": 2.6660899653979238, + "measurement_wall_seconds": 20.016, + "client_output_tokens": 1538, + "server_output_tokens": 1541, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 6.656 + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 118.67889540432174, + "server_steps_per_s": 43.30175913400928, + "server_spec_accept_length": 2.7407407407407405, + "measurement_wall_seconds": 20.016, + "client_output_tokens": 2368, + "server_output_tokens": 2368, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 8.719 + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 171.3898781157694, + "server_steps_per_s": 60.189597231174496, + "server_spec_accept_length": 2.8475, + "measurement_wall_seconds": 20, + "client_output_tokens": 3417, + "server_output_tokens": 3417, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 12.266 + }, + { + "context_tokens": 16384, + "concurrency": 2, + "aggregate_tps": 75.65, + "server_steps_per_s": 28.000000000000004, + "server_spec_accept_length": 2.7017857142857142, + "measurement_wall_seconds": 20, + "client_output_tokens": 1513, + "server_output_tokens": 1513, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 7.11 + }, + { + "context_tokens": 16384, + "concurrency": 4, + "aggregate_tps": 117.38194201055911, + "server_steps_per_s": 42.26551154305115, + "server_spec_accept_length": 2.7772511848341233, + "measurement_wall_seconds": 20, + "client_output_tokens": 2344, + "server_output_tokens": 2344, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 9.687 + }, + { + "context_tokens": 16384, + "concurrency": 8, + "aggregate_tps": 184.4835363105187, + "server_steps_per_s": 63.74981205840255, + "server_spec_accept_length": 2.893867924528302, + "measurement_wall_seconds": 20.015, + "client_output_tokens": 3681, + "server_output_tokens": 3681, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 11.797 + } + ] + }, + { + "id": "122857", + "source_sha256": "05d846465153230f4312a59aefef12c499d81634855f3c84cb6757baf653c0e4", + "metadata": { + "version": "0.4.32", + "engine": "vllm", + "model": "glm-5.3-flash-spark", + "decode_mode": "duration", + "duration_per_test": 20, + "decode_warmup_seconds": 5, + "decode_warmup_context": 16384, + "decode_warmup_concurrency": 1, + "cell_warmup_timeout_seconds": 900, + "max_tokens": 2048, + "temperature": 1, + "ignore_eos": true, + "dcp_size": 4, + "concurrency_levels": [ + 1, + 2, + 4, + 8 + ], + "context_lengths": [ + 8192, + 16384 + ], + "prefill_mode": "integrated_decode_scout" + }, + "prefill": { + "8192": { + "ttft_seconds": 3.079, + "prefill_seconds": 3.079, + "tok_per_sec": 2661, + "client_ttft_seconds": 3.079, + "client_tok_per_sec": 2661, + "prompt_tokens": 8192, + "samples": 1, + "method": "integrated_scout", + "server_validation": { + "method": "prometheus:kv_computed", + "tok_per_sec": 2694, + "prefill_seconds": 3.041, + "prompt_tokens": 8192, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 1, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "16384": { + "ttft_seconds": 6.031, + "prefill_seconds": 6.031, + "tok_per_sec": 2717, + "client_ttft_seconds": 6.031, + "client_tok_per_sec": 2717, + "prompt_tokens": 16384, + "samples": 1, + "method": "integrated_scout", + "server_validation": { + "method": "prometheus:kv_computed", + "tok_per_sec": 2728, + "prefill_seconds": 6.006, + "prompt_tokens": 16384, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 1, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "65536": { + "ttft_seconds": 23.579, + "prefill_seconds": 23.579, + "tok_per_sec": 2779, + "client_ttft_seconds": 23.579, + "client_tok_per_sec": 2779, + "prompt_tokens": 65536, + "samples": 1, + "method": "scout_only", + "server_validation": { + "method": "", + "tok_per_sec": 0, + "prefill_seconds": 0, + "prompt_tokens": 0, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 0, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "131072": { + "ttft_seconds": 47.516, + "prefill_seconds": 47.516, + "tok_per_sec": 2758, + "client_ttft_seconds": 47.516, + "client_tok_per_sec": 2758, + "prompt_tokens": 131072, + "samples": 1, + "method": "scout_only", + "server_validation": { + "method": "", + "tok_per_sec": 0, + "prefill_seconds": 0, + "prompt_tokens": 0, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 0, + "invalid_reason": "" + }, + "hardware_summary": {} + } + }, + "decode": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 49.35, + "server_steps_per_s": 18.7, + "server_spec_accept_length": 2.63903743315508, + "measurement_wall_seconds": 20, + "client_output_tokens": 987, + "server_output_tokens": 987, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 12.39 + }, + { + "context_tokens": 16384, + "concurrency": 1, + "aggregate_tps": 51.830337022278336, + "server_steps_per_s": 18.52871951521061, + "server_spec_accept_length": 2.7972972972972974, + "measurement_wall_seconds": 20, + "client_output_tokens": 1035, + "server_output_tokens": 1035, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 13.953 + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 74.65599199404768, + "server_steps_per_s": 28.22116587442553, + "server_spec_accept_length": 2.6453900709219855, + "measurement_wall_seconds": 20, + "client_output_tokens": 1492, + "server_output_tokens": 1492, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 6.578 + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 123.99137974179853, + "server_steps_per_s": 43.903172455058815, + "server_spec_accept_length": 2.8242009132420094, + "measurement_wall_seconds": 20, + "client_output_tokens": 2474, + "server_output_tokens": 2474, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 10.125 + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 164.26744244341373, + "server_steps_per_s": 61.39338917579799, + "server_spec_accept_length": 2.6756535947712417, + "measurement_wall_seconds": 20, + "client_output_tokens": 3275, + "server_output_tokens": 3275, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 13.36 + }, + { + "context_tokens": 16384, + "concurrency": 2, + "aggregate_tps": 78.62580128207695, + "server_steps_per_s": 28.645833333342686, + "server_spec_accept_length": 2.744755244755245, + "measurement_wall_seconds": 20, + "client_output_tokens": 1570, + "server_output_tokens": 1570, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 7.141 + }, + { + "context_tokens": 16384, + "concurrency": 4, + "aggregate_tps": 120.54040530406225, + "server_steps_per_s": 44.03302476860722, + "server_spec_accept_length": 2.7375, + "measurement_wall_seconds": 20, + "client_output_tokens": 2409, + "server_output_tokens": 2409, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 9.703 + }, + { + "context_tokens": 16384, + "concurrency": 8, + "aggregate_tps": 168.41787743246968, + "server_steps_per_s": 60.730983862215936, + "server_spec_accept_length": 2.77317880794702, + "measurement_wall_seconds": 20, + "client_output_tokens": 3350, + "server_output_tokens": 3350, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 11.89 + } + ] + }, + { + "id": "123541", + "source_sha256": "9664bc0410db423d3edc759106f10bc24a0fe1d15952102cdd74913ff96c2149", + "metadata": { + "version": "0.4.32", + "engine": "vllm", + "model": "glm-5.3-flash-spark", + "decode_mode": "duration", + "duration_per_test": 20, + "decode_warmup_seconds": 5, + "decode_warmup_context": 16384, + "decode_warmup_concurrency": 1, + "cell_warmup_timeout_seconds": 900, + "max_tokens": 2048, + "temperature": 1, + "ignore_eos": true, + "dcp_size": 4, + "concurrency_levels": [ + 1, + 2, + 4, + 8 + ], + "context_lengths": [ + 8192, + 16384 + ], + "prefill_mode": "integrated_decode_scout" + }, + "prefill": { + "8192": { + "ttft_seconds": 3.062, + "prefill_seconds": 3.062, + "tok_per_sec": 2675, + "client_ttft_seconds": 3.062, + "client_tok_per_sec": 2675, + "prompt_tokens": 8192, + "samples": 1, + "method": "integrated_scout", + "server_validation": { + "method": "prometheus:kv_computed", + "tok_per_sec": 2697, + "prefill_seconds": 3.038, + "prompt_tokens": 8192, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 1, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "16384": { + "ttft_seconds": 6.063, + "prefill_seconds": 6.063, + "tok_per_sec": 2702, + "client_ttft_seconds": 6.063, + "client_tok_per_sec": 2702, + "prompt_tokens": 16384, + "samples": 1, + "method": "integrated_scout", + "server_validation": { + "method": "prometheus:kv_computed", + "tok_per_sec": 2721, + "prefill_seconds": 6.022, + "prompt_tokens": 16384, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 1, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "65536": { + "ttft_seconds": 23.609, + "prefill_seconds": 23.609, + "tok_per_sec": 2776, + "client_ttft_seconds": 23.609, + "client_tok_per_sec": 2776, + "prompt_tokens": 65536, + "samples": 1, + "method": "scout_only", + "server_validation": { + "method": "", + "tok_per_sec": 0, + "prefill_seconds": 0, + "prompt_tokens": 0, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 0, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "131072": { + "ttft_seconds": 47.672, + "prefill_seconds": 47.672, + "tok_per_sec": 2749, + "client_ttft_seconds": 47.672, + "client_tok_per_sec": 2749, + "prompt_tokens": 131072, + "samples": 1, + "method": "scout_only", + "server_validation": { + "method": "", + "tok_per_sec": 0, + "prefill_seconds": 0, + "prompt_tokens": 0, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 0, + "invalid_reason": "" + }, + "hardware_summary": {} + } + }, + "decode": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 52.792233786884466, + "server_steps_per_s": 18.815052041581573, + "server_spec_accept_length": 2.8058510638297873, + "measurement_wall_seconds": 20.015, + "client_output_tokens": 1055, + "server_output_tokens": 1055, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 11.938 + }, + { + "context_tokens": 16384, + "concurrency": 1, + "aggregate_tps": 52.93204466912869, + "server_steps_per_s": 18.97941809801303, + "server_spec_accept_length": 2.788918205804749, + "measurement_wall_seconds": 20, + "client_output_tokens": 1057, + "server_output_tokens": 1057, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 13.39 + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 77.08114068067856, + "server_steps_per_s": 29.268781636876643, + "server_spec_accept_length": 2.633561643835616, + "measurement_wall_seconds": 20, + "client_output_tokens": 1538, + "server_output_tokens": 1538, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 7.187 + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 115.25, + "server_steps_per_s": 41.8, + "server_spec_accept_length": 2.757177033492823, + "measurement_wall_seconds": 20, + "client_output_tokens": 2305, + "server_output_tokens": 2305, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 8.203 + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 173.45, + "server_steps_per_s": 61.076754890678934, + "server_spec_accept_length": 2.8398692810457513, + "measurement_wall_seconds": 20, + "client_output_tokens": 3469, + "server_output_tokens": 3476, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 12.219 + }, + { + "context_tokens": 16384, + "concurrency": 2, + "aggregate_tps": 76.45, + "server_steps_per_s": 29.000000000000004, + "server_spec_accept_length": 2.636206896551724, + "measurement_wall_seconds": 20, + "client_output_tokens": 1529, + "server_output_tokens": 1529, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 7.578 + }, + { + "context_tokens": 16384, + "concurrency": 4, + "aggregate_tps": 118.96395944170077, + "server_steps_per_s": 43.16835659065935, + "server_spec_accept_length": 2.755813953488372, + "measurement_wall_seconds": 20, + "client_output_tokens": 2370, + "server_output_tokens": 2370, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 9.219 + }, + { + "context_tokens": 16384, + "concurrency": 8, + "aggregate_tps": 171.9245283018868, + "server_steps_per_s": 61.18238993710692, + "server_spec_accept_length": 2.810032894736842, + "measurement_wall_seconds": 20, + "client_output_tokens": 3417, + "server_output_tokens": 3417, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 13.766 + } + ] + }, + { + "id": "124222", + "source_sha256": "4213df0fc47ba53308ea7dc076e3e1c77e9097e5eaef95db4e09ca4cd404e434", + "metadata": { + "version": "0.4.32", + "engine": "", + "model": "glm-5.3-flash-spark", + "decode_mode": "duration", + "duration_per_test": 20, + "decode_warmup_seconds": 5, + "decode_warmup_context": 16384, + "decode_warmup_concurrency": 1, + "cell_warmup_timeout_seconds": 900, + "max_tokens": 2048, + "temperature": 1, + "ignore_eos": true, + "dcp_size": 4, + "concurrency_levels": [ + 1, + 2, + 4, + 8 + ], + "context_lengths": [ + 8192, + 16384 + ], + "prefill_mode": "integrated_decode_scout" + }, + "prefill": { + "8192": { + "ttft_seconds": 3.062, + "prefill_seconds": 3.062, + "tok_per_sec": 2675, + "client_ttft_seconds": 3.062, + "client_tok_per_sec": 2675, + "prompt_tokens": 8192, + "samples": 1, + "method": "integrated_scout", + "server_validation": { + "method": "prometheus:kv_computed", + "tok_per_sec": 2697, + "prefill_seconds": 3.038, + "prompt_tokens": 8192, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 1, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "16384": { + "ttft_seconds": 6.063, + "prefill_seconds": 6.063, + "tok_per_sec": 2702, + "client_ttft_seconds": 6.063, + "client_tok_per_sec": 2702, + "prompt_tokens": 16384, + "samples": 1, + "method": "integrated_scout", + "server_validation": { + "method": "prometheus:kv_computed", + "tok_per_sec": 2725, + "prefill_seconds": 6.013, + "prompt_tokens": 16384, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 1, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "65536": { + "ttft_seconds": 23.578, + "prefill_seconds": 23.578, + "tok_per_sec": 2780, + "client_ttft_seconds": 23.578, + "client_tok_per_sec": 2780, + "prompt_tokens": 65536, + "samples": 1, + "method": "scout_only", + "server_validation": { + "method": "", + "tok_per_sec": 0, + "prefill_seconds": 0, + "prompt_tokens": 0, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 0, + "invalid_reason": "" + }, + "hardware_summary": {} + }, + "131072": { + "ttft_seconds": 48.344, + "prefill_seconds": 48.344, + "tok_per_sec": 2711, + "client_ttft_seconds": 48.344, + "client_tok_per_sec": 2711, + "prompt_tokens": 131071, + "samples": 1, + "method": "scout_only", + "server_validation": { + "method": "", + "tok_per_sec": 0, + "prefill_seconds": 0, + "prompt_tokens": 0, + "request_prompt_tokens": 0, + "cached_tokens": 0, + "token_source": "", + "samples": 0, + "invalid_reason": "" + }, + "hardware_summary": {} + } + }, + "decode": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 52.8, + "server_steps_per_s": 19.05, + "server_spec_accept_length": 2.7716535433070866, + "measurement_wall_seconds": 20, + "client_output_tokens": 1056, + "server_output_tokens": 1056, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 11.844 + }, + { + "context_tokens": 16384, + "concurrency": 1, + "aggregate_tps": 47.012390088058716, + "server_steps_per_s": 18.984812150331894, + "server_spec_accept_length": 2.4763157894736842, + "measurement_wall_seconds": 20.016, + "client_output_tokens": 941, + "server_output_tokens": 941, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 13.984 + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 78.9, + "server_steps_per_s": 29.600000000000005, + "server_spec_accept_length": 2.6655405405405403, + "measurement_wall_seconds": 20, + "client_output_tokens": 1578, + "server_output_tokens": 1578, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 7.141 + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 123.18949531411437, + "server_steps_per_s": 44.10364356241686, + "server_spec_accept_length": 2.793181818181818, + "measurement_wall_seconds": 20.016, + "client_output_tokens": 2458, + "server_output_tokens": 2458, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 9.156 + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 168.77003205133715, + "server_steps_per_s": 60.89743589745578, + "server_spec_accept_length": 2.771381578947368, + "measurement_wall_seconds": 20.015, + "client_output_tokens": 3370, + "server_output_tokens": 3370, + "aggregate_source": "openai_continuous_usage", + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false, + "warmup_duration": 12.813 + } + ] + } + ] +} diff --git a/recipes/glm53-spark-mtp3-managed-mesh-tp4.json b/recipes/glm53-spark-mtp3-managed-mesh-tp4.json index 94d0e43f..546de0db 100644 --- a/recipes/glm53-spark-mtp3-managed-mesh-tp4.json +++ b/recipes/glm53-spark-mtp3-managed-mesh-tp4.json @@ -36,8 +36,8 @@ "image_receipt": "runtime/glm53-spark-mtp3-mesh/image-receipt.json", "compute_contract": "runtime/glm53-spark-mtp3-mesh/compute/source-lock.json", "compute_contract_sha256": "2a444f7c1ad4319f64afbffb16403491a4863934372cf818f18c510d58c0d00e", - "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86", - "image_id": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", + "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:b65d427f9be49c97d57e404ad1a6118769c1119df876a8944c1f186a6b380c5d", + "image_id": "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", "site_template": "runtime/glm53-spark-mtp3-mesh/site.example.json", "fabric_template": "runtime/glm53-spark-mtp3-mesh/fabric.example.json", "renderer": "runtime/glm53-spark-mtp3-mesh/profile.py", @@ -104,7 +104,7 @@ "identity_contract": "runtime/glm53-spark-mtp3-mesh/pins.json#/cache_identity" }, "transport": { - "bundle_manifest_sha256": "4204fabc93303226b9a120b094ef3c82ed4aadd1d7f97cfbe291204c027ed45f", + "bundle_manifest_sha256": "69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500", "routing_contract": "spark_transport/experiments/glm53_rocenante_overlay/overlay_contract.json", "captured_sircl_query_rows": [16, 20, 24, 28, 32], "large_eager_prefill": "dual-rail fused SIRCL", diff --git a/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md b/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md index cc9296b9..41b90bb4 100644 --- a/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md +++ b/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md @@ -30,14 +30,14 @@ to the tested Linux/ARM64 image. Pull before using its local image ID: ```bash set -euo pipefail -mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86' -mtp_image_id='sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df' +mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:b65d427f9be49c97d57e404ad1a6118769c1119df876a8944c1f186a6b380c5d' +mtp_image_id='sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766' docker pull "$mtp_image" test "$(docker image inspect "$mtp_image" --format '{{.Id}}')" = "$mtp_image_id" ``` The immutable reference is also published as tag -`glm53-spark-mtp3-nvfp4-a16-2a444f7c`; use the digest above for deployment. +`glm53-spark-mtp3-stream69313e19`; use the digest above for deployment. The [compute-image equivalence record](compute-image-equivalence.json) verifies that all 4,891 vLLM, 385 B12X, and 150 SparkCache package files and the selected environment match tested private image `sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74`. diff --git a/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json b/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json index 3d2de485..02579b98 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json +++ b/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json @@ -2,7 +2,7 @@ "schema": "sparkring-compute-image-equivalence/v1", "checks_passed": true, "tested_serving_image": "sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74", - "published_image": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", + "published_image": "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", "package_files": { "b12x": 385, "sparkcache": 150, @@ -22,5 +22,5 @@ "VLLM_MTP_NVFP4_LM_HEAD": "1", "VLLM_MXFP8_LM_HEAD": "0" }, - "scope": "Installed package files excluding Python bytecode and selected environment. Public image construction and native-marker verification passed. Full-model serving was measured on the tested serving image; no claim of a separate four-rank serving run on the published image." + "scope": "Installed compute package files excluding Python bytecode and selected environment. The mounted RoCEnante transport overlay additionally contains capture-guard and shared-staging ordering fixes; it is not included in compute package equivalence. Public image construction and native-marker verification passed. Full-model serving was measured on the tested serving image; no separate four-rank serving run is claimed for this published image." } diff --git a/runtime/glm53-spark-mtp3-mesh/image-receipt.json b/runtime/glm53-spark-mtp3-mesh/image-receipt.json index 8a9e21e2..9d0fcc8b 100644 --- a/runtime/glm53-spark-mtp3-mesh/image-receipt.json +++ b/runtime/glm53-spark-mtp3-mesh/image-receipt.json @@ -1,15 +1,15 @@ { "added_layers": 8, - "bundle_manifest_sha256": "4204fabc93303226b9a120b094ef3c82ed4aadd1d7f97cfbe291204c027ed45f", + "bundle_manifest_sha256": "69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500", "checks_passed": true, - "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache:glm53-spark-mtp3-nvfp4-a16-2a444f7c", - "image_id": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", - "image_reference": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", - "image_size_bytes": 22107698916, + "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache:glm53-spark-mtp3-stream69313e19", + "image_id": "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", + "image_reference": "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", + "image_size_bytes": 22107699822, "inside_image": { "b12x_commit": "b58f34eaf978277621efced6678e6713fd7122e4", "bundle_files": 28, - "bundle_manifest_sha256": "4204fabc93303226b9a120b094ef3c82ed4aadd1d7f97cfbe291204c027ed45f", + "bundle_manifest_sha256": "69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500", "checks_passed": true, "compute": { "b12x_files": 385, @@ -50,7 +50,7 @@ }, "rocenante_lazy_import": "/opt/spark-sircl/b12x_overlay/b12x/comm/roce/__init__.py", "sircl_native_sha256": "61aa0ec56a1b438439bed8611dab0353d2c72c10af02bbd917fb77c87b33e5fc", - "source_receipt_sha256": "79dedb68ef65f51d99d66f8664e9235aa6fe8444205bb4c223e5046d076ab3c6", + "source_receipt_sha256": "09f3d010db1e15d8c5bb414cd75a7aebbcdf4e6a73d2224e65933cfc3348e3c4", "sparkcache_commit": "66057174301a4759ca3a45207ea41016689449cb", "status": "research-only", "vllm_commit": "e02b174693e13859de61811b5e8cd13d5308e259", @@ -61,7 +61,7 @@ "parent_layers_retained": 81, "platform": "linux/arm64", "schema": "sparkring-mtp3-mesh-image-receipt/v1", - "source_receipt_sha256": "79dedb68ef65f51d99d66f8664e9235aa6fe8444205bb4c223e5046d076ab3c6", + "source_receipt_sha256": "09f3d010db1e15d8c5bb414cd75a7aebbcdf4e6a73d2224e65933cfc3348e3c4", "status": "research-only", "verification_command": [ "docker", @@ -86,7 +86,7 @@ "PYTHONDONTWRITEBYTECODE=1", "--entrypoint", "python3", - "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", + "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", "-I", "/opt/sparkring/bin/verify-mtp3-mesh-image.py", "--inside-image" diff --git a/runtime/glm53-spark-mtp3-mesh/pins.json b/runtime/glm53-spark-mtp3-mesh/pins.json index 5cb80eac..9039b84a 100644 --- a/runtime/glm53-spark-mtp3-mesh/pins.json +++ b/runtime/glm53-spark-mtp3-mesh/pins.json @@ -20,9 +20,9 @@ "speculation": {"method": "mtp", "num_speculative_tokens": 3, "attention_backend": "B12X", "draft_tensor_parallel_size": 4, "draft_sample_method": "probabilistic", "rejection_sample_method": "standard"}, "capture_sizes": [4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64], "captured_sircl_query_rows": [16, 20, 24, 28, 32], - "canonical_bundle_manifest_sha256": "4204fabc93303226b9a120b094ef3c82ed4aadd1d7f97cfbe291204c027ed45f", + "canonical_bundle_manifest_sha256": "69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500", "measured_bundle_manifest_sha256": "701bdc42069a97492981b8f34e006ebfa9e68c2160472cba631b56965efae226", - "bundle_difference": "Configuration/manifest line endings differ; parsed configuration and every executable file are identical.", + "bundle_difference": "The serving measurements use the recorded measured bundle. The canonical bundle additionally preserves capture-ID stream guards and orders shared staging/output copies across streams; routing and collective algorithms are unchanged.", "marker": { "source": "../../spark_transport/experiments/cx7_hairpin_diagonal/native/mlx5_rdma_tx_rewrite_probe.c", "source_sha256": "8684a6961b8e86aa474fa2310ff71e4cdf219a63a72ceb5593b2f95e54812792", diff --git a/runtime/glm53-spark-mtp3-mesh/profile.py b/runtime/glm53-spark-mtp3-mesh/profile.py index 258383ad..7cdef692 100644 --- a/runtime/glm53-spark-mtp3-mesh/profile.py +++ b/runtime/glm53-spark-mtp3-mesh/profile.py @@ -127,6 +127,21 @@ def load_image_receipt(path: Path) -> dict: "helper_sha256": sha(BASE / "warmup_dflash.py"), "temperature": 1.0, }: raise ValueError("Image receipt does not verify the sampling warmup helper") + compute = inside.get("compute") + required = PINS.get("compute", {}) + if required: + if not isinstance(compute, dict): + raise ValueError("Image receipt lacks the required compute attestation") + for field in ("source_lock_sha256", "b12x_revision", "b12x_tree", "cuda_version"): + if compute.get(field) != required[field]: + raise ValueError(f"Image receipt compute identity differs: {field}") + lock = json.loads((HERE / required["source_lock"]).read_text()) + environment = compute.get("environment", {}) + if (not isinstance(environment, dict) + or any(environment.get(k) != v for k, v in lock["environment"].items()) + or compute.get("proposal_head_nvfp4") is not True + or compute.get("target_head_quantization") is not False): + raise ValueError("Image receipt does not attest the required proposal and verifier paths") return document diff --git a/runtime/glm53-spark-mtp3-mesh/public-image.json b/runtime/glm53-spark-mtp3-mesh/public-image.json index 0fe33ffb..183fccad 100644 --- a/runtime/glm53-spark-mtp3-mesh/public-image.json +++ b/runtime/glm53-spark-mtp3-mesh/public-image.json @@ -6,10 +6,10 @@ "anonymous_config_read": true, "anonymous_pull": true, "anonymous_pull_method": "Docker pull on the build host with an empty client credential directory; local image layers already present", - "public_reference": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86", - "tag": "glm53-spark-mtp3-nvfp4-a16-2a444f7c", - "manifest_digest": "sha256:1b97e1dc9cb93c39f887f40bab24359a9b6ec998c28d2417b160f2103cd5fd86", - "config_image_id": "sha256:dd6c51efaf4127df863ac85c3be3fe46f260b34c7ab2deb384669fffdbe857df", + "public_reference": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:b65d427f9be49c97d57e404ad1a6118769c1119df876a8944c1f186a6b380c5d", + "tag": "glm53-spark-mtp3-stream69313e19", + "manifest_digest": "sha256:b65d427f9be49c97d57e404ad1a6118769c1119df876a8944c1f186a6b380c5d", + "config_image_id": "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", "platform": "linux/arm64", "layer_count": 89, "all_layer_diff_ids_match_tested_image": true, diff --git a/runtime/glm53-spark-mtp3-mesh/qualification/test_checks.py b/runtime/glm53-spark-mtp3-mesh/qualification/test_checks.py index a70ed499..e043ce57 100644 --- a/runtime/glm53-spark-mtp3-mesh/qualification/test_checks.py +++ b/runtime/glm53-spark-mtp3-mesh/qualification/test_checks.py @@ -33,7 +33,8 @@ def inputs(tmp_path): "image_id": "sha256:" + "a" * 64, "image_reference": "sha256:" + "a" * 64, "bundle_manifest_sha256": bundle_sha, "source_receipt_sha256": "b" * 64, "inside_image": {"checks_passed": True, "bundle_manifest_sha256": bundle_sha, - "source_receipt_sha256": "b" * 64, "cuda_initialized": False, "model_loaded": False}} + "source_receipt_sha256": "b" * 64, "cuda_initialized": False, "model_loaded": False, + "compute": json.loads((HERE.parent / "image-receipt.json").read_text())["inside_image"]["compute"]}} receipt_path = tmp_path / "image.json" receipt_path.write_text(json.dumps(receipt)) plan = {"schema": "sparkring-mtp3-mesh-render/v1", "image": receipt, diff --git a/runtime/glm53-spark-mtp3-mesh/test_profile.py b/runtime/glm53-spark-mtp3-mesh/test_profile.py index 237a364a..c94543d6 100644 --- a/runtime/glm53-spark-mtp3-mesh/test_profile.py +++ b/runtime/glm53-spark-mtp3-mesh/test_profile.py @@ -330,10 +330,24 @@ def _image_receipt_document(): "image_id": "sha256:" + "a" * 64, "image_reference": "sha256:" + "a" * 64, "bundle_manifest_sha256": bundle_sha, "source_receipt_sha256": source_sha, "inside_image": {"checks_passed": True, "bundle_manifest_sha256": bundle_sha, - "source_receipt_sha256": source_sha, "cuda_initialized": False, "model_loaded": False}, + "source_receipt_sha256": source_sha, "cuda_initialized": False, "model_loaded": False, + "compute": json.loads((mesh_profile.HERE / "image-receipt.json").read_text())["inside_image"]["compute"]}, } +@pytest.mark.parametrize('field', ['compute', 'source_lock_sha256', 'b12x_revision', 'environment', 'proposal_head_nvfp4', 'target_head_quantization']) +def test_receipt_requires_profile_compute(tmp_path, field): + document = _image_receipt_document() + if field == 'compute': + del document['inside_image']['compute'] + else: + document['inside_image']['compute'][field] = None + path = tmp_path / 'wrong-compute.json' + path.write_text(json.dumps(document)) + with pytest.raises(ValueError): + mesh_profile.load_image_receipt(path) + + def test_verified_image_receipt_changes_only_image_selection(tmp_path, manifest_bundle): site_path = _site(tmp_path) document = _image_receipt_document() diff --git a/spark_transport/experiments/glm53_rocenante_overlay/rocenante_vllm_overlay.py b/spark_transport/experiments/glm53_rocenante_overlay/rocenante_vllm_overlay.py index 1a0acfd8..068209d2 100644 --- a/spark_transport/experiments/glm53_rocenante_overlay/rocenante_vllm_overlay.py +++ b/spark_transport/experiments/glm53_rocenante_overlay/rocenante_vllm_overlay.py @@ -435,8 +435,7 @@ def all_reduce(self, tensor: Any) -> Any: try: if capturing: stream = torch.cuda.current_stream(self.device) - with self._runtime.capture(stream=stream): - result = self._runtime.all_reduce(tensor, stream=stream) + result = self._runtime.all_reduce(tensor, stream=stream) self._captured_nodes += 1 else: result = self._runtime.all_reduce(tensor) diff --git a/spark_transport/experiments/glm53_rocenante_overlay/test_stream_lifetime.py b/spark_transport/experiments/glm53_rocenante_overlay/test_stream_lifetime.py new file mode 100644 index 00000000..e6b4fb14 --- /dev/null +++ b/spark_transport/experiments/glm53_rocenante_overlay/test_stream_lifetime.py @@ -0,0 +1,81 @@ +"""Exercise stream admission and shared staging order without CUDA hardware.""" +import ast +from contextlib import contextmanager, nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[3] +SOURCE = ROOT / 'third_party/b12x_roce/b12x/comm/roce/roce_oneshot.py' + + +def methods(names, namespace): + tree = ast.parse(SOURCE.read_text()) + nodes = [node for cls in tree.body if isinstance(cls, ast.ClassDef) + for node in cls.body if isinstance(node, ast.FunctionDef) and node.name in names] + assert len(nodes) == len(names) + exec(compile(ast.Module(body=nodes, type_ignores=[]), str(SOURCE), 'exec'), namespace) + return namespace + + +def test_per_call_capture_context_cannot_reset_cuda_capture_identity(): + state = SimpleNamespace(stream='A', capture_id=123) + namespace = methods({'capture', '_order_stream'}, { + 'contextmanager':contextmanager, 'Optional':__import__('typing').Optional, + 'torch':SimpleNamespace(cuda=SimpleNamespace(current_stream=lambda device:state.stream)), + '_capture_id':lambda stream:state.capture_id}) + runtime = SimpleNamespace(device=0, _capture_id=0, _capture_stream=None, _last_stream=None) + with namespace['capture'](runtime): + namespace['_order_stream'](runtime, True) + state.stream = 'B' + with pytest.raises(RuntimeError, match='one stream'): + with namespace['capture'](runtime): + namespace['_order_stream'](runtime, True) + state.capture_id = 124 + with namespace['capture'](runtime): + namespace['_order_stream'](runtime, True) + assert runtime._capture_stream == 'B' + + +def test_misaligned_input_waits_before_shared_scratch_copy(): + events = [] + class Tensor: + shape = (4, 4096) + dtype = 'bf16' + device = 0 + def __init__(self, address): self.address = address + def is_contiguous(self): return True + def data_ptr(self): return self.address + def numel(self): return 4 * 4096 + def element_size(self): return 2 + def copy_(self, other): events.append('copy') + cuda = SimpleNamespace(device=lambda _:nullcontext(), is_current_stream_capturing=lambda:False) + namespace = methods({'all_reduce'}, { + 'torch':SimpleNamespace(Tensor=Tensor,cuda=cuda), + 'Optional':__import__('typing').Optional, 'Sequence':__import__('typing').Sequence, + '_nullcontext':nullcontext, 'PACK_BYTES':16, + 'is_launcher_prepared':lambda *args:True, + 'get_launcher':lambda *args:lambda *a:events.append('launch')}) + runtime = SimpleNamespace(_lock=nullcontext(), device=0, check_health=lambda:None, + should_allreduce=lambda inp:True, _launcher_key=lambda dtype:(), + _aligned_scratch=lambda which,like:Tensor(32), + _order_stream=lambda capturing:events.append('wait'), + _mark_stream=lambda capturing:events.append('record'), + _recv_base=0,_flag_base=0,_send_base=0,_ctrl_base=0,_slot_bytes=0, + _epoch_address=0,spin_limit=1,_blocks=1) + namespace['all_reduce'](runtime, Tensor(2), out=Tensor(18)) + assert events == ['wait','copy','launch','copy','record'] + + +def test_padded_gather_orders_staging_and_records_after_output_copy(): + tree = ast.parse(SOURCE.read_text()) + method = next(n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name=='all_gather') + calls = [n for n in ast.walk(method) if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)] + # Select the padded path after the aligned path's early return. + stage = next(n for n in calls if n.func.attr=='copy_' and isinstance(n.func.value, ast.Subscript)) + waits = [n.lineno for n in calls if n.func.attr=='_order_stream'] + records = [n.lineno for n in calls if n.func.attr=='_mark_stream'] + output = max(n.lineno for n in calls if n.func.attr=='copy_') + assert max(waits) < stage.lineno + assert max(records) > output diff --git a/third_party/b12x_roce/README.md b/third_party/b12x_roce/README.md index f8ae6169..a2120a67 100644 --- a/third_party/b12x_roce/README.md +++ b/third_party/b12x_roce/README.md @@ -10,6 +10,10 @@ unpublished Git ref because the complete selected source is included here. The bundle builder verifies its digest before copying it. `LICENSE` contains the upstream Apache-2.0 license; source copyright notices are retained. +Local stream-safety modifications preserve CUDA capture-ID admission across +Python context boundaries. Eager calls wait for the preceding operation before +writing shared staging buffers, and record completion after output copies. + ## Attribution and design origins RoCEnante originates with Local Inference Lab's contributors, not SparkRing. diff --git a/third_party/b12x_roce/b12x/comm/roce/roce_oneshot.py b/third_party/b12x_roce/b12x/comm/roce/roce_oneshot.py index 84d2c7ec..2ee48e7d 100644 --- a/third_party/b12x_roce/b12x/comm/roce/roce_oneshot.py +++ b/third_party/b12x_roce/b12x/comm/roce/roce_oneshot.py @@ -561,15 +561,17 @@ def all_reduce( "RoCE all-reduce launcher must be prepared before CUDA graph capture" ) launcher = get_launcher(*key) - if out is None: - out = torch.empty_like(inp) - src = inp + if out is None: + out = torch.empty_like(inp) + # Shared staging buffers belong to the preceding operation until + # its completion event, including its output copy, has fired. + self._order_stream(capturing) + src = inp if inp.data_ptr() % PACK_BYTES != 0: src = self._aligned_scratch(0, inp) src.copy_(inp) dst = out if out.data_ptr() % PACK_BYTES == 0 else self._aligned_scratch(1, out) - self._order_stream(capturing) - launcher( + launcher( src.data_ptr(), dst.data_ptr(), nbytes // PACK_BYTES, @@ -760,24 +762,26 @@ def all_gather( # requested layout. nbytes = inp.numel() * inp.element_size() padded = _align_up(nbytes, PACK_BYTES) - staged, gathered = self._gather_scratch(padded) - staged[:nbytes].copy_(inp.reshape(-1).view(torch.uint8)) - self._order_stream(capturing) + staged, gathered = self._gather_scratch(padded) + self._order_stream(capturing) + staged[:nbytes].copy_(inp.reshape(-1).view(torch.uint8)) self._launch_gather( staged.data_ptr(), gathered.data_ptr(), padded, padded // PACK_BYTES ) - self._mark_stream(capturing) - stacked = ( + stacked = ( gathered.view(self.world_size, padded)[:, :nbytes] .reshape(-1) .view(inp.dtype) .reshape(self.world_size, *inp.shape) ) - result = stacked.movedim(0, dim).reshape(shape) - if out is None: - return result.contiguous() - out.copy_(result) - return out + result = stacked.movedim(0, dim).reshape(shape) + if out is None: + result = result.contiguous() + else: + out.copy_(result) + result = out + self._mark_stream(capturing) + return result def _gather_scratch(self, padded: int) -> tuple[torch.Tensor, torch.Tensor]: """Fixed device scratch for the padded all-gather path, allocated once. @@ -838,15 +842,13 @@ def capture(self, stream: object = None, *, channel_id: Optional[str] = None): capture began. Inside, every collective must use one stream. """ - self._last_stream = None - self._capture_stream = None - self._capture_id = 0 - try: - yield self - finally: - self._capture_stream = None - self._capture_id = 0 - self._last_stream = None + self._last_stream = None + # CUDA's capture ID owns stream admission, not this Python context. + # Nested or per-call contexts must retain the same-capture guard. + try: + yield self + finally: + self._last_stream = None # -- diagnostics / lifecycle -------------------------------------------------- diff --git a/third_party/b12x_roce/provenance.json b/third_party/b12x_roce/provenance.json index 70e4e88c..f6deddfb 100644 --- a/third_party/b12x_roce/provenance.json +++ b/third_party/b12x_roce/provenance.json @@ -5,7 +5,8 @@ "upstream_base_commit": "ffb7442d04a9f50b950df1fb17280acad881b7d5", "license": "Apache-2.0", "scope": "Only b12x.comm.roce; no model kernels or other B12X package modules", - "roce_tree_sha256": "3acb9c3cda49931f7988b9b426bd1304df029de95d868a90346df2d09ed8f627", + "roce_tree_sha256": "902a9dfd1a9c8ec379b002b13737701dd6bc58e240abcb5ab9b443455961a3a4", + "local_changes": ["Preserve capture-ID stream admission across Python contexts", "Order shared input staging and output copies across streams"], "git_state": { "commit": "eac260a8257cc6b14e7d4ad674f51e9a09b8790f", "roce_source_dirty": false, From 322083fd06fa3227a552ddf512379c56bc4e24e1 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:34:21 -0500 Subject: [PATCH 04/16] List native MTP3 mesh in the GLM-5.3 profile table --- README.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 72eee353..cf279057 100644 --- a/README.md +++ b/README.md @@ -69,18 +69,24 @@ together. In the tables, 1M means 1,048,576 tokens. ### GLM-5.3 Flash -These three **DFlash2/SIRCL profiles** use the GLM-5.3 Flash NVFP4 target, -an external BF16 DFlash2 predictor at depth seven, FP8 KV, B12X kernels, and -the same ARM64 image. Target verification captures use rows 8 through 128 in -eight-row increments, covering full request batches from C1 through C16. +Choose native MTP3 with hardware-forwarded mesh for the NVFP4-Spark +checkpoint, or an external DFlash2 predictor with SIRCL for the NVFP4 +checkpoint. Each quickstart supplies its own pinned image and setup steps. | Profile | Deployment | Context | Seqs | Batch | KV / cache | Approx. recorded KV capacity | Start here | |---|---|---:|---:|---:|---|---:|---| -| DCP1 | 4 Sparks · TP4/DCP1 | 1M | 16 | 8,192 | FP8 · 24 GiB/rank; SparkCache enabled | ~1.30M tokens | [Quickstart](docs/GLM53_JJ_R8_GB10_SPARKCACHE_TP4_QUICKSTART.md) | -| DCP2 | 4 Sparks · TP4/DCP2 | 1M | 16 | 8,192 | FP8 · 24 GiB/rank; SparkCache enabled | ~2.90M tokens | [Quickstart](docs/GLM53_JJ_R8_GB10_SPARKCACHE_TP4_QUICKSTART.md) | -| **DCP4 preferred** | **4 Sparks · TP4/DCP4** | **1M** | **16** | **8,192** | **FP8 · 24 GiB/rank; SparkCache enabled** | **~4.32M tokens** | **[Quickstart](docs/GLM53_JJ_R8_GB10_SPARKCACHE_TP4_QUICKSTART.md)** | +| **NVFP4-Spark · native MTP3 · mesh** (research-only) | **4 Sparks · TP4/DCP4** | **1M** | **16** | **8,192** | **FP8 · 24 GiB/rank; SparkCache enabled** | Reported at startup | **[Mesh quickstart](docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md)** | +| DFlash2/SIRCL · DCP1 | 4 Sparks · TP4/DCP1 | 1M | 16 | 8,192 | FP8 · 24 GiB/rank; SparkCache enabled | ~1.30M tokens | [Quickstart](docs/GLM53_JJ_R8_GB10_SPARKCACHE_TP4_QUICKSTART.md) | +| DFlash2/SIRCL · DCP2 | 4 Sparks · TP4/DCP2 | 1M | 16 | 8,192 | FP8 · 24 GiB/rank; SparkCache enabled | ~2.90M tokens | [Quickstart](docs/GLM53_JJ_R8_GB10_SPARKCACHE_TP4_QUICKSTART.md) | +| DFlash2/SIRCL · DCP4 (preferred DFlash2 profile) | 4 Sparks · TP4/DCP4 | 1M | 16 | 8,192 | FP8 · 24 GiB/rank; SparkCache enabled | ~4.32M tokens | [Quickstart](docs/GLM53_JJ_R8_GB10_SPARKCACHE_TP4_QUICKSTART.md) | -All three default to 24 GiB of KV memory per rank. The reference capacities +#### DFlash2/SIRCL configuration + +The three DFlash2 profiles use an external BF16 predictor at depth seven, +FP8 KV, B12X kernels, and the same ARM64 image. Target verification captures +use rows 8 through 128 in eight-row increments, covering C1 through C16. + +All profiles above default to 24 GiB of KV memory per rank. The DFlash2 reference capacities were measured at 26/30/24 GiB for DCP1/2/4 respectively; vLLM reports the actual model-wide capacity at startup. That capacity is shared across requests. **DCP4 is the preferred DFlash2 profile** and the documented asynchronous @@ -109,10 +115,6 @@ for its terms. The native-MTP3 profile below does not require these weights. ### GLM-5.3 Flash Spark with native MTP3 and mesh transport -| Profile | Deployment | Context | Seqs | Batch | KV / cache | Start here | -|---|---|---:|---:|---:|---|---| -| NVFP4-Spark + native MTP3 + NVFP4/BF16 proposal head + mesh · research-only | 4 Sparks · TP4/DCP4 | 1M | 16 | 8,192 | FP8 · 24 GiB/rank; SparkCache enabled | [Quickstart](docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md) | - This profile uses the NVFP4-Spark checkpoint's built-in three-token predictor; no external draft checkpoint or DFlash model is required. A separate runtime- NVFP4 proposal head uses BF16 activations while the target/verifier head retains From 504c6b0f96410dd37607e77f1ea3da56765d1eb3 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:20:57 -0500 Subject: [PATCH 05/16] Describe MTP3 profile behavior and evidence by artifact identity --- README.md | 9 +++--- docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md | 30 ++++++++++--------- ...spark-mtp3-nvfp4-proposal-head-20260905.md | 13 ++++---- .../glm53-spark-mtp3-managed-mesh-tp4.json | 2 +- runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md | 2 +- runtime/glm53-spark-mtp3-mesh/README.md | 30 +++++++++++-------- .../glm53-spark-mtp3-mesh/compute/README.md | 11 ++++--- runtime/glm53-spark-mtp3-mesh/test_image.py | 10 +++---- 8 files changed, 58 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index cf279057..9c3b25e1 100644 --- a/README.md +++ b/README.md @@ -126,10 +126,11 @@ four-row increments through 64 rows. The image composition uses CUDA 13.3, the native-MTP3 metadata port derived from Local Inference Lab vLLM revision `3512b066`, and the complete B12X tree at `b58f34ea` with vLLM integration based on `a8c796f3`. The complete B12X update -also includes MoE and dense-precision work, so comparisons against the previous -public image cannot attribute a gain to dense kernels alone. The +also includes MoE and dense-precision work, so comparisons across different +compute configurations cannot attribute a gain to dense kernels alone. The [head-specific comparison](performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md) -uses a control with the same metadata+dense+B12X composition. +uses a control with the same CUDA version, B12X kernels, metadata reuse, and +dense-kernel integration; only the proposal-head configuration differs. The [profile package](runtime/glm53-spark-mtp3-mesh/README.md) provides the public image, transport files, and temperature-one warmup. The @@ -180,7 +181,7 @@ The native-MTP3 proposal-head row uses three observations per decode cell and three prefill scouts per listed context. Relative to two shared-BF16-head controls with the same compute composition, C1 changed by +8.22% raw output throughput and +4.90% normalized sequence steps/s; C2/C4/C8 were mixed and -prefill was flat within 0.36%. The previous image's +prefill was flat within 0.36%. The image-specific [consolidated report](performance/records/glm53-flash/spark-mtp3-validation-summary-20260905.md) retains the 32K/64K matrix, Estonia **30/30** at C8, and **4/4** needle-hunt checks through 507,367 prompt tokens, with the image identity recorded for diff --git a/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md b/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md index cc09f649..965c4b77 100644 --- a/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md +++ b/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md @@ -4,18 +4,18 @@ Status: **research-only**. The profile's composition, managed host service, and CPU checks are **implemented**. The [managed functional record](../performance/records/glm53-flash/spark-mtp3-managed-mesh-functional-20260905.md) qualifies bounded installer, policy-scoped fault/recovery, post-recovery -readiness, and one persistent-cache recall case for the published managed -image identified below. +readiness, and one persistent-cache recall case for the image digest recorded +in that report. Broader cache/failure coverage and unattended serving remain unqualified. The [public application-install record](../performance/records/glm53-flash/spark-mtp3-public-application-install-20260905.md) covers fresh public checkouts, extracted image artifacts, empty application caches, installation, native correctness, and model-restart cache restoration on four prepared hosts. It does not qualify a factory-reset OS/network setup. -Both functional records are image-specific to the earlier published -composition. The NVFP4/BF16 proposal-head performance record does not transfer -their restart, cache, or failure-containment qualification to the current -public image ID; repeat those checks before describing it as qualified. +These functional records qualify only their recorded image digests. Restart, +cache restoration, and failure containment require validation for the image +pinned in `runtime/glm53-spark-mtp3-mesh/public-image.json`; proposal-head +throughput measurements do not establish those properties. **Starting with four stock Sparks and no image?** Follow [the managed-mesh prerequisite section](PREREQUISITES.md#four-spark-managed-hardware-forwarded-mesh) @@ -74,7 +74,8 @@ for completed checks, repeat counts, and the remaining test plan. The [profile results table](../runtime/glm53-spark-mtp3-mesh/README.md#operator-benchmark-observations) shows the completed three-run C1/C2/C4/C8 screen. At 8K, aggregate decode means were **51.6, 76.9, 120.8, and 168.8 tok/s**. Against two shared-BF16-head -controls using the same metadata+dense+B12X base, C1 improved **8.22% raw** and +controls using the same CUDA version, B12X kernels, metadata reuse, and dense-kernel +integration, C1 improved **8.22% raw** and **4.90% in normalized sequence steps/s**. Higher concurrency was mixed and prefill means were flat within 0.36% over 8K–128K. The linked record provides the receipt hashes, exact settings, and limitations. @@ -368,18 +369,19 @@ the dedicated namespace `glm53-spark-df116c4f-mtp3-nvfp4-a16-b58f34ea-mesh4204fabc-tail-cow-v2`; shared-BF16-head and external-DFlash entries must not be renamed into it. The `draft_policy=separate` field describes cache registration layout, not an -external draft model. The linked functional record -includes an uncached publication and stopped-container restoration under the -previous image's identity. Persistent restoration under the new namespace is -unqualified until the same stopped-container check passes. The earlier record -covers one recall prompt and does not qualify other checkpoints, all context -lengths, or concurrent cache workloads. +external draft model. The +[managed functional record](../performance/records/glm53-flash/spark-mtp3-managed-mesh-functional-20260905.md) +includes an uncached publication and stopped-container restoration for its +recorded image and namespace. Restoration under the NVFP4-proposal-head +namespace named above is research-only until a stopped-container restore test +passes for that configuration. The linked record covers one recall prompt, +not all context lengths or concurrent cache workloads. ## Obtain the image and target Pull the published Linux/ARM64 managed image on every Spark. No local build is required. The [registry receipt](../runtime/glm53-spark-mtp3-mesh/public-image.json) -records anonymous access and its match to the tested image. The separate +records anonymous access and the published image's manifest/config identities. The separate [content receipt](../runtime/glm53-spark-mtp3-mesh/image-receipt.json) is the input accepted by the renderer, installer, and native qualification runner. Keep both with the checkout; do not substitute `public-image.json` for the diff --git a/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md b/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md index 5d633d34..7efe5072 100644 --- a/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md +++ b/performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md @@ -26,7 +26,7 @@ which records CUDA 13.3.33, complete B12X revision port from Local Inference Lab vLLM revision `3512b066e7796128c0c380ccc558182960f2f0ea`, and dense wrapper changes from revision `a8c796f3af74106b2d8d441e9ec54588936a5388`. The complete B12X tree also -contains MoE and other September changes, so these runs do not isolate a dense +contains MoE and other package changes in B12X revision `b58f34ea`, so these runs do not isolate a dense kernel contribution. The changed variable was a separate runtime-NVFP4 proposal head with BF16 @@ -34,11 +34,12 @@ activations. The target/verifier head kept its BF16 checkpoint representation. The proposal head adds 85.08 MiB of persistent packed weight and scale storage per rank while the retained BF16 target head remains allocated. It is not a net 85.08 MiB model-memory reduction. The shared-BF16-head control used the same -metadata+dense+B12X base without the separate proposal allocation. -Thus the proposal-head comparison preserves the verifier implementation. The -complete compute image still differs from the previous public image in CUDA, -metadata, dense, MoE, and B12X code; “BF16 verifier retained” is not a claim -that every target-side kernel is unchanged across those images. +CUDA version, B12X kernels, metadata reuse, and dense-kernel integration without +the separate proposal allocation. Thus the proposal-head comparison preserves +the verifier implementation. Comparisons against the separate +[mesh matrix](spark-mtp3-mesh-20260905.md) also differ in CUDA, metadata, +dense-kernel integration, MoE, and B12X code; retaining a BF16 verifier head +does not imply that every target-side kernel matches between those configurations. The benchmark used harness version 0.4.32, temperature 1.0, ignored EOS, a 20-second sustained-decode window, 8,192-token decode context, and concurrency diff --git a/recipes/glm53-spark-mtp3-managed-mesh-tp4.json b/recipes/glm53-spark-mtp3-managed-mesh-tp4.json index 546de0db..7d289a7f 100644 --- a/recipes/glm53-spark-mtp3-managed-mesh-tp4.json +++ b/recipes/glm53-spark-mtp3-managed-mesh-tp4.json @@ -121,7 +121,7 @@ "record": "performance/records/glm53-flash/spark-mtp3-managed-mesh-functional-20260905.md", "proposal_head_performance_record": "performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md", "status": "research-only", - "scope": "The proposal-head performance screen covers the recorded private image. Managed lifecycle and persistent-recall qualification remain bound to the previous published image.", + "scope": "Proposal-head performance was measured on serving image 04d5a35b, identified by the full digest in the proposal-head record. Managed lifecycle and persistent-recall results qualify only the image digests and cache namespaces in their linked functional records.", "limitations": [ "This opt-in profile does not replace the recommended DFlash/SIRCL profile.", "The published compute image has build/content equivalence to the tested private image but requires fresh native, startup, restart and persistent-cache receipts before those qualifications transfer.", diff --git a/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md b/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md index 41b90bb4..d027596f 100644 --- a/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md +++ b/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md @@ -26,7 +26,7 @@ The site plan and native-MTP3 launch configuration remain separate inputs. ## Published image The [public registry receipt](public-image.json) binds the published manifest -to the tested Linux/ARM64 image. Pull before using its local image ID: +to its Linux/ARM64 config-image identity. Pull before using its local image ID: ```bash set -euo pipefail diff --git a/runtime/glm53-spark-mtp3-mesh/README.md b/runtime/glm53-spark-mtp3-mesh/README.md index 88c1efb2..4a681fba 100644 --- a/runtime/glm53-spark-mtp3-mesh/README.md +++ b/runtime/glm53-spark-mtp3-mesh/README.md @@ -50,34 +50,40 @@ mean aggregate output tokens per second across those requests. |---:|---:|---:|---:|---:| | 8K | 51.6 | 76.9 | 120.8 | 168.8 | -Relative to the same metadata+dense+B12X composition with a shared BF16 -proposal head, C1 improved 8.22% in raw output throughput and 4.90% in +Relative to a shared-BF16-proposal-head control with the same CUDA version, +B12X kernels, metadata reuse, and dense-kernel integration, C1 improved 8.22% +in raw output throughput and 4.90% in acceptance-normalized sequence steps/s. C2/C4/C8 results were mixed. Repeated prefill means moved by no more than 0.36% over 8K–128K contexts, so no prefill -gain is claimed. The earlier +gain is claimed. The [broader matrix](../../performance/records/glm53-flash/spark-mtp3-mesh-20260905.md) -belongs to the previous image configuration and remains historical context. +records measurements for its explicitly identified image and serving settings. The head-specific control preserves the verifier implementation; the complete CUDA 13.3/B12X `b58f34ea` image changes other target computation relative to -the previous public image, so cross-image gains cannot be assigned only to the +the configuration recorded in that matrix, so cross-image gains cannot be assigned only to the proposal head or dense kernels. The published image's [compute-equivalence record](compute-image-equivalence.json) matches every vLLM, B12X, and SparkCache package file and selected environment entry to the -tested private image. That supports applying the recorded compute result to the -published bytes. Native transport, model startup, restart, and persistent-cache -checks remain image-ID-specific and have not been repeated on the public image. +serving image identified in that record. This establishes compute-package +content equivalence, not end-to-end performance equivalence: the mounted +transport differs outside that comparison. Throughput belongs to measured +serving image `04d5a35b`; the full digest is in the equivalence record. Native +transport, model startup, restart, and persistent-cache checks remain +image-specific and have not been repeated on published image `69c794bf`. The separate [Estonia accuracy record](../../performance/records/glm53-flash/spark-mtp3-country-recall-20260905.md) reports **30/30 correct** at C8 on one repeated 133,208-token prompt, no output-limit hits, and 1.96 s mean cache-primed TTFT. Its 23.8 tok/s figure -uses summed request times, not cluster wall time. Both records retain the -operator screenshots and metric definitions. +uses summed request times, not cluster wall time. The Estonia record includes +the operator screenshot and metric definitions. The [long-context needle hunt](../../performance/records/glm53-flash/spark-mtp3-needle-20260905.md) passed **4/4** exact-value, revision, and cross-reference checks, reaching -**507,367 actual prompt tokens** on the published image. +**507,367 actual prompt tokens** on serving image +`sha256:26273b8e358df139ae913610a5d43084ff0fd08aafe282ef633a3bc74afefe47`, +as recorded in that report. It is not a measurement of image `69c794bf`. ## Composition @@ -163,7 +169,7 @@ serving lifecycle. Use authenticated managed readiness for serving. Native MTP uses the target checkpoint as the draft identity. The profile sets SparkCache's `draft_policy=separate` because that describes the registered state layout; it does not request an external model. A dedicated namespace -includes `mtp3-nvfp4-a16-b58f34ea` so the new compute and proposal-head +includes `mtp3-nvfp4-a16-b58f34ea` so the NVFP4-proposal-head compute composition cannot restore shared-BF16-head or external-DFlash entries. Do not relabel those entries to avoid cache misses. Persistent restore under the native-MTP identity requires its own qualification. diff --git a/runtime/glm53-spark-mtp3-mesh/compute/README.md b/runtime/glm53-spark-mtp3-mesh/compute/README.md index ba35aa58..b5c0efd4 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/README.md +++ b/runtime/glm53-spark-mtp3-mesh/compute/README.md @@ -26,7 +26,7 @@ while network access is available. The prepared directory contains the pinned B12X source and CUDA archives. Docker copies that directory into the build and runs `apply_compute.py` with network access disabled. The installer verifies the parent hashes before extracting the replacement archive; this preserves the -mixed line endings of the tested source without requiring Git in the image. +mixed line endings bound by the source lock without requiring Git in the image. `verify_compute.py` requires exact installed hashes and rejects missing or partial source maps. @@ -39,8 +39,7 @@ revision `3512b066e7796128c0c380ccc558182960f2f0ea`, with dense-kernel integrati from revision `a8c796f3af74106b2d8d441e9ec54588936a5388`; vLLM is licensed under Apache License 2.0. -B12X source archives use LF endings. The tested ARM64 image was assembled from -a Windows checkout and contains CRLF bytes for Python and C source files. The -preparation step performs that deterministic byte conversion so the public -image can be compared exactly with the tested image. Markdown and compressed -profile data retain the archive bytes. +B12X source archives use LF endings. Source preparation converts Python and C +files to CRLF to reproduce the installed package hashes in `source-lock.json`. +Markdown and compressed profile data retain the archive bytes. This byte-level +contract makes package-content verification independent of checkout settings. diff --git a/runtime/glm53-spark-mtp3-mesh/test_image.py b/runtime/glm53-spark-mtp3-mesh/test_image.py index 0d2ba490..d1c094ec 100644 --- a/runtime/glm53-spark-mtp3-mesh/test_image.py +++ b/runtime/glm53-spark-mtp3-mesh/test_image.py @@ -135,14 +135,14 @@ def test_layered_file_map_rejects_unbound_or_wrong_base_override(tmp_path): ) -def test_complete_package_file_map_rejects_stale_parent_module(tmp_path): +def test_complete_package_file_map_rejects_unmanifested_module(tmp_path): package = tmp_path / "b12x" package.mkdir() - current = package / "current.py" - current.write_text("VALUE = 1\n", encoding="utf-8") - records = {"b12x/current.py": verifier.sha256(current)} + declared = package / "declared.py" + declared.write_text("VALUE = 1\n", encoding="utf-8") + records = {"b12x/declared.py": verifier.sha256(declared)} assert verifier.verify_complete_package_file_map(tmp_path, "b12x", records) == 1 - (package / "stale_parent.py").write_text("VALUE = 0\n", encoding="utf-8") + (package / "unmanifested.py").write_text("VALUE = 0\n", encoding="utf-8") with pytest.raises(ValueError, match="complete manifest"): verifier.verify_complete_package_file_map(tmp_path, "b12x", records) From 3226102365696e8ec829fcc28ed7e56888b6c1af Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:38:49 -0500 Subject: [PATCH 06/16] Compose GLM loader ownership, independent RNG and B12X selector sources --- THIRD_PARTY_NOTICES.md | 22 +- docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md | 19 +- .../spark-mtp3-compute-matrices-20260905.json | 1465 ++++++ .../spark-mtp3-compute-matrices-20260905.md | 62 + runtime/glm53-spark-mtp3-mesh/Dockerfile | 2 +- runtime/glm53-spark-mtp3-mesh/README.md | 6 + .../glm53-spark-mtp3-mesh/compute/README.md | 21 +- .../compute/b12x-selector-files.tar.gz | Bin 0 -> 54422 bytes .../compute/prepare_compute_source.py | 33 + .../compute/source-lock.json | 182 +- .../compute/test_compute.py | 45 +- .../compute/vllm-compute-files.tar.gz | Bin 99559 -> 152792 bytes .../compute/vllm-e02-to-compute.patch | 4193 +++-------------- runtime/glm53-spark-mtp3-mesh/pins.json | 42 +- runtime/glm53-spark-mtp3-mesh/test_image.py | 2 +- 15 files changed, 2442 insertions(+), 3652 deletions(-) create mode 100644 performance/records/glm53-flash/spark-mtp3-compute-matrices-20260905.json create mode 100644 performance/records/glm53-flash/spark-mtp3-compute-matrices-20260905.md create mode 100644 runtime/glm53-spark-mtp3-mesh/compute/b12x-selector-files.tar.gz diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 71f50c51..1fbe44b8 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -58,11 +58,14 @@ SparkRing implementation. ## 3. vLLM (referenced, patched, and selected source included) `runtime/glm53-spark-mtp3-mesh/compute/vllm-compute-files.tar.gz` includes -fourteen vLLM Python files for GLM metadata reuse, dense-kernel integration, -and the NVFP4 proposal head. The adjacent patch provides a readable diff; +24 vLLM Python files for GLM metadata reuse, dense-kernel integration, +the NVFP4 proposal head, deferred-weight ownership, and independent draft +and rejection-sampling randomness. The adjacent patch provides a readable diff; `source-lock.json` records the base, donor revisions, and exact file hashes. These files derive from Local Inference Lab's vLLM fork at `3512b066` and -`a8c796f3`, under Apache-2.0 with their contributor notices retained. +`a8c796f3`, loader correction `17e341b9`, and independent-RNG backport +`44e6766e` from [PR 653](https://github.com/local-inference-lab/vllm/pull/653), +under Apache-2.0 with their contributor notices retained. The unified diffs under `runtime/deepseek0731-gb10/patches/` contain context and removed lines from vLLM, pinned to the source revision recorded by that @@ -296,3 +299,16 @@ The native-MTP3 profile references Operators must obtain and use that checkpoint under its own license and notices. Native MTP uses its included prediction layer and does not require an external DFlash checkpoint. + +## 12. B12X selector source and MoE scale sharing + +The native-MTP3 compute package downloads B12X revision +`ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301`, including its shared native NVFP4 +scales for A4/A16 MoE paths. Three selector Python files are included in +`runtime/glm53-spark-mtp3-mesh/compute/b12x-selector-files.tar.gz` from +[B12X PR 316](https://github.com/local-inference-lab/b12x/pull/316), revision +`9ac142824b4edb750892a0fb63d914230086495d`. They implement the top-k-512 +candidate buffer, exact overflow handling, and omission of unused terminal +scores. These files are licensed under Apache-2.0; source notices are retained +and the downloaded B12X archive supplies the license. The compute source lock +binds the source archive and each base/result file hash. diff --git a/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md b/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md index cc09f649..25a94970 100644 --- a/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md +++ b/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md @@ -60,14 +60,23 @@ The proposal-head and metadata implementation is derived from [Local Inference Lab vLLM revision `3512b066`](https://github.com/local-inference-lab/vllm/commit/3512b066e7796128c0c380ccc558182960f2f0ea), as retained in [revision `a8c796f3`](https://github.com/local-inference-lab/vllm/commit/a8c796f3af74106b2d8d441e9ec54588936a5388). -The packaged compute source uses the complete Local Inference Lab B12X tree at -[revision `b58f34ea`](https://github.com/local-inference-lab/b12x/commit/b58f34eaf978277621efced6678e6713fd7122e4). -That tree includes MoE and -dense-precision changes in addition to the head kernel. SparkRing does not -claim an isolated dense-kernel result for this composition. +The compute source uses Local Inference Lab B12X +[revision `ef308bac`](https://github.com/local-inference-lab/b12x/commit/ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301) +for shared native MoE scale storage, with three source-checked selector files +from [PR 316](https://github.com/local-inference-lab/b12x/pull/316). The vLLM +composition also includes deferred-weight ownership from `17e341b9` and +independent draft/rejection randomness from +[PR 653](https://github.com/local-inference-lab/vllm/pull/653). Exact source and +file identities are in the compute source lock. ## Recorded benchmark observations +The [compute matrices](../performance/records/glm53-flash/spark-mtp3-compute-matrices-20260905.md) +compare proposal-head, loader/RNG, scale-sharing, and selector configurations +at C1/C2/C4/C8/C12/C16. They include averages across 8K/32K/64K context rows +and source-hashed individual cells. Their compute images and transport bundle +are identified separately from the combined image requiring qualification. + See the [consolidated validation report](../performance/records/glm53-flash/spark-mtp3-validation-summary-20260905.md) for completed checks, repeat counts, and the remaining test plan. diff --git a/performance/records/glm53-flash/spark-mtp3-compute-matrices-20260905.json b/performance/records/glm53-flash/spark-mtp3-compute-matrices-20260905.json new file mode 100644 index 00000000..ed512880 --- /dev/null +++ b/performance/records/glm53-flash/spark-mtp3-compute-matrices-20260905.json @@ -0,0 +1,1465 @@ +{ + "schema": "sparkring-glm53-compute-matrices/v1", + "status": "research-only", + "aggregation": "Arithmetic mean over three context rows per concurrency; each context has equal weight.", + "contexts": [ + 8192, + 32768, + 65536 + ], + "concurrencies": [ + 1, + 2, + 4, + 8, + 12, + 16 + ], + "normalized_definition": "Aggregate sequence steps/s, not batched engine iterations/s.", + "runs": [ + { + "configuration": "Public mesh MTP3", + "image": null, + "source_receipt_name": "glm-5.3-flash-spark-dcp4-dflash7-bf16-SIRCL-and-MESH-20260904-233405.json", + "source_receipt_sha256": "f0916f6b72cb8256225169b44c4f11e3ca764a5dd854977b8963686197b843fa", + "metadata": { + "version": "0.4.32", + "model": "glm-5.3-flash-spark", + "duration_per_test": 20.0, + "temperature": 1.0, + "max_tokens": 2048, + "dcp_size": 4 + }, + "cells": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 48.186139604737214, + "server_steps_per_s": 17.813360020027464, + "server_spec_accept_length": 2.705056179775281, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 1, + "aggregate_tps": 49.9, + "server_steps_per_s": 17.75, + "server_spec_accept_length": 2.8112676056338026, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 1, + "aggregate_tps": 43.01558752984392, + "server_steps_per_s": 17.635891286916262, + "server_spec_accept_length": 2.4390934844192635, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 75.84311766170075, + "server_steps_per_s": 27.479390457137953, + "server_spec_accept_length": 2.76, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 112.22394711780265, + "server_steps_per_s": 43.066753467786825, + "server_spec_accept_length": 2.605813953488372, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 168.80835257585548, + "server_steps_per_s": 59.83335006554259, + "server_spec_accept_length": 2.821308724832215, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 12, + "aggregate_tps": 193.35819391325268, + "server_steps_per_s": 69.54243096176405, + "server_spec_accept_length": 2.7804347826086957, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 16, + "aggregate_tps": 231.3372852411072, + "server_steps_per_s": 81.9853310561318, + "server_spec_accept_length": 2.8216911764705883, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 2, + "aggregate_tps": 76.8423682237785, + "server_steps_per_s": 27.17961528851463, + "server_spec_accept_length": 2.827205882352941, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 4, + "aggregate_tps": 118.97960206497459, + "server_steps_per_s": 41.69799027719413, + "server_spec_accept_length": 2.8533653846153846, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 8, + "aggregate_tps": 164.56330128210502, + "server_steps_per_s": 59.69551282053231, + "server_spec_accept_length": 2.7567114093959733, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 12, + "aggregate_tps": 197.31954623009523, + "server_steps_per_s": 68.66780443723486, + "server_spec_accept_length": 2.8735380116959064, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 16, + "aggregate_tps": 222.73572110305662, + "server_steps_per_s": 78.76626312349859, + "server_spec_accept_length": 2.8278061224489797, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 2, + "aggregate_tps": 76.37566402715933, + "server_steps_per_s": 28.064548461423374, + "server_spec_accept_length": 2.7214285714285715, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 4, + "aggregate_tps": 119.02971984175005, + "server_steps_per_s": 42.499874705601705, + "server_spec_accept_length": 2.8007075471698113, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 8, + "aggregate_tps": 165.85, + "server_steps_per_s": 59.78371883448483, + "server_spec_accept_length": 2.774166666666667, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 12, + "aggregate_tps": 192.30188679245282, + "server_steps_per_s": 69.43396226415094, + "server_spec_accept_length": 2.769565217391304, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 16, + "aggregate_tps": 220.88050314465409, + "server_steps_per_s": 78.8930817610063, + "server_spec_accept_length": 2.799744897959184, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + } + ], + "prefill": { + "8192": { + "ttft_seconds": 3.031, + "tok_per_sec": 2703.0, + "samples": 1, + "method": "integrated_scout" + }, + "32768": { + "ttft_seconds": 11.797, + "tok_per_sec": 2778.0, + "samples": 1, + "method": "integrated_scout" + }, + "65536": { + "ttft_seconds": 23.516, + "tok_per_sec": 2787.0, + "samples": 1, + "method": "integrated_scout" + }, + "131072": { + "ttft_seconds": 47.39, + "tok_per_sec": 2766.0, + "samples": 1, + "method": "scout_only" + } + }, + "column_averages": { + "1": { + "aggregate_tps": 47.03390904486038, + "server_steps_per_s": 17.733083768981242, + "aggregate_tps_vs_public_percent": 0.0, + "server_steps_per_s_vs_public_percent": 0.0 + }, + "2": { + "aggregate_tps": 76.35371663754619, + "server_steps_per_s": 27.57451806902532, + "aggregate_tps_vs_public_percent": 0.0, + "server_steps_per_s_vs_public_percent": 0.0 + }, + "4": { + "aggregate_tps": 116.74442300817576, + "server_steps_per_s": 42.42153948352755, + "aggregate_tps_vs_public_percent": 0.0, + "server_steps_per_s_vs_public_percent": 0.0 + }, + "8": { + "aggregate_tps": 166.4072179526535, + "server_steps_per_s": 59.77086057351991, + "aggregate_tps_vs_public_percent": 0.0, + "server_steps_per_s_vs_public_percent": 0.0 + }, + "12": { + "aggregate_tps": 194.32654231193357, + "server_steps_per_s": 69.21473255438329, + "aggregate_tps_vs_public_percent": 0.0, + "server_steps_per_s_vs_public_percent": 0.0 + }, + "16": { + "aggregate_tps": 224.9845031629393, + "server_steps_per_s": 79.8815586468789, + "aggregate_tps_vs_public_percent": 0.0, + "server_steps_per_s_vs_public_percent": 0.0 + } + } + }, + { + "configuration": "NVFP4 MTP proposal head", + "image": "sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74", + "source_receipt_name": "glm-5.3-flash-spark-dcp4-MTP3-SIRCL-and-MESH-r2420260905-124723.json", + "source_receipt_sha256": "29e86dbbd0522cd9c42151b828826fbc8b48a938ef86ba86d0cd0f428af30bf3", + "metadata": { + "version": "0.4.32", + "model": "glm-5.3-flash-spark", + "duration_per_test": 20.0, + "temperature": 1.0, + "max_tokens": 2048, + "dcp_size": 4 + }, + "cells": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 50.0, + "server_steps_per_s": 18.9, + "server_spec_accept_length": 2.6455026455026456, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 1, + "aggregate_tps": 42.05, + "server_steps_per_s": 18.4, + "server_spec_accept_length": 2.2853260869565215, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 1, + "aggregate_tps": 49.877309830134514, + "server_steps_per_s": 18.829185237078892, + "server_spec_accept_length": 2.648936170212766, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 77.62031148263905, + "server_steps_per_s": 28.8447092993549, + "server_spec_accept_length": 2.6909722222222223, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 122.38147739784979, + "server_steps_per_s": 43.299589054767495, + "server_spec_accept_length": 2.826388888888889, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 163.18348118094238, + "server_steps_per_s": 60.14133213056844, + "server_spec_accept_length": 2.7133333333333334, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 12, + "aggregate_tps": 200.8827807590449, + "server_steps_per_s": 70.42182876047416, + "server_spec_accept_length": 2.8525641025641026, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 16, + "aggregate_tps": 225.35779076752496, + "server_steps_per_s": 82.18403168508102, + "server_spec_accept_length": 2.7421116504854366, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 2, + "aggregate_tps": 75.15, + "server_steps_per_s": 29.5, + "server_spec_accept_length": 2.5474576271186438, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 4, + "aggregate_tps": 122.28737533215585, + "server_steps_per_s": 43.10128802690738, + "server_spec_accept_length": 2.837209302325581, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 8, + "aggregate_tps": 167.0177550399394, + "server_steps_per_s": 61.390309960626375, + "server_spec_accept_length": 2.7205882352941178, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 12, + "aggregate_tps": 204.43374460863902, + "server_steps_per_s": 72.22389407174686, + "server_spec_accept_length": 2.8305555555555557, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 16, + "aggregate_tps": 223.45, + "server_steps_per_s": 80.29690905047809, + "server_spec_accept_length": 2.7827970297029703, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 2, + "aggregate_tps": 79.75, + "server_steps_per_s": 29.2, + "server_spec_accept_length": 2.731164383561644, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 4, + "aggregate_tps": 120.53683208949174, + "server_steps_per_s": 43.06675346778683, + "server_spec_accept_length": 2.7988372093023255, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 8, + "aggregate_tps": 175.2551405152696, + "server_steps_per_s": 61.53536775407056, + "server_spec_accept_length": 2.8480392156862746, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 12, + "aggregate_tps": 194.6, + "server_steps_per_s": 70.43803680981594, + "server_spec_accept_length": 2.7627118644067794, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 16, + "aggregate_tps": 225.81132075471697, + "server_steps_per_s": 80.50314465408805, + "server_spec_accept_length": 2.8049999999999997, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + } + ], + "prefill": { + "8192": { + "ttft_seconds": 3.062, + "tok_per_sec": 2675.0, + "samples": 1, + "method": "integrated_scout" + }, + "32768": { + "ttft_seconds": 11.875, + "tok_per_sec": 2759.0, + "samples": 1, + "method": "integrated_scout" + }, + "65536": { + "ttft_seconds": 23.609, + "tok_per_sec": 2776.0, + "samples": 1, + "method": "integrated_scout" + }, + "131072": { + "ttft_seconds": 47.625, + "tok_per_sec": 2752.0, + "samples": 1, + "method": "scout_only" + } + }, + "column_averages": { + "1": { + "aggregate_tps": 47.309103276711504, + "server_steps_per_s": 18.70972841235963, + "aggregate_tps_vs_public_percent": 0.5850975124960245, + "aggregate_tps_vs_preceding_percent": 0.5850975124960245, + "server_steps_per_s_vs_public_percent": 5.507472113151235, + "server_steps_per_s_vs_preceding_percent": 5.507472113151235 + }, + "2": { + "aggregate_tps": 77.50677049421301, + "server_steps_per_s": 29.181569766451634, + "aggregate_tps_vs_public_percent": 1.5101476489224774, + "aggregate_tps_vs_preceding_percent": 1.5101476489224774, + "server_steps_per_s_vs_public_percent": 5.828031856816129, + "server_steps_per_s_vs_preceding_percent": 5.828031856816129 + }, + "4": { + "aggregate_tps": 121.73522827316579, + "server_steps_per_s": 43.15587684982057, + "aggregate_tps_vs_public_percent": 4.2749838805066664, + "aggregate_tps_vs_preceding_percent": 4.2749838805066664, + "server_steps_per_s_vs_public_percent": 1.7310483665454068, + "server_steps_per_s_vs_preceding_percent": 1.7310483665454068 + }, + "8": { + "aggregate_tps": 168.48545891205046, + "server_steps_per_s": 61.02233661508846, + "aggregate_tps_vs_public_percent": 1.248888711058349, + "aggregate_tps_vs_preceding_percent": 1.248888711058349, + "server_steps_per_s_vs_public_percent": 2.093789564948967, + "server_steps_per_s_vs_preceding_percent": 2.093789564948967 + }, + "12": { + "aggregate_tps": 199.9721751225613, + "server_steps_per_s": 71.02791988067898, + "aggregate_tps_vs_public_percent": 2.905229899868922, + "aggregate_tps_vs_preceding_percent": 2.905229899868922, + "server_steps_per_s_vs_public_percent": 2.619655179438918, + "server_steps_per_s_vs_preceding_percent": 2.619655179438918 + }, + "16": { + "aggregate_tps": 224.87303717408065, + "server_steps_per_s": 80.99469512988239, + "aggregate_tps_vs_public_percent": -0.049543851817168694, + "aggregate_tps_vs_preceding_percent": -0.049543851817168694, + "server_steps_per_s_vs_public_percent": 1.3934836799118777, + "server_steps_per_s_vs_preceding_percent": 1.3934836799118777 + } + } + }, + { + "configuration": "Proposal head + loader/RNG fixes", + "image": "sha256:e3e83b5ef49c8787ae1a1b4945c22bb55f3432d43d74fdc1b1c3564d702bc2d5", + "source_receipt_name": "glm-5.3-flash-spark-dcp4-MTP3-SIRCL-and-MESH-r2420260905-145256.json", + "source_receipt_sha256": "bab411558cc190e3d19bc33d9ee45444f11975c81c32283e1c4dc1ce33ab4037", + "metadata": { + "version": "0.4.32", + "model": "glm-5.3-flash-spark", + "duration_per_test": 20.0, + "temperature": 1.0, + "max_tokens": 2048, + "dcp_size": 4 + }, + "cells": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 47.43557668254504, + "server_steps_per_s": 18.764073054804207, + "server_spec_accept_length": 2.528, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 1, + "aggregate_tps": 51.541232986247394, + "server_steps_per_s": 18.76501200955609, + "server_spec_accept_length": 2.7466666666666666, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 1, + "aggregate_tps": 49.27637838639795, + "server_steps_per_s": 18.691040077599222, + "server_spec_accept_length": 2.6363636363636367, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 78.73502731426919, + "server_steps_per_s": 29.569488297529485, + "server_spec_accept_length": 2.6627118644067798, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 123.08001204685665, + "server_steps_per_s": 43.77070575238948, + "server_spec_accept_length": 2.811926605504587, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 178.71103538113528, + "server_steps_per_s": 63.34569509864133, + "server_spec_accept_length": 2.8212025316455698, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 12, + "aggregate_tps": 200.3524672710372, + "server_steps_per_s": 72.50755287014164, + "server_spec_accept_length": 2.7631944444444443, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 16, + "aggregate_tps": 228.58294698602262, + "server_steps_per_s": 82.24148357169068, + "server_spec_accept_length": 2.7794117647058822, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 2, + "aggregate_tps": 80.0600450338313, + "server_steps_per_s": 30.022516887686738, + "server_spec_accept_length": 2.666666666666667, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 4, + "aggregate_tps": 120.52721258879146, + "server_steps_per_s": 43.90097223608371, + "server_spec_accept_length": 2.7454337899543377, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 8, + "aggregate_tps": 172.28783026373765, + "server_steps_per_s": 60.83101091774244, + "server_spec_accept_length": 2.832236842105263, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 12, + "aggregate_tps": 201.13442425438606, + "server_steps_per_s": 71.67955024588552, + "server_spec_accept_length": 2.8060224089635852, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 16, + "aggregate_tps": 233.04766200459304, + "server_steps_per_s": 81.79221169709588, + "server_spec_accept_length": 2.8492647058823533, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 2, + "aggregate_tps": 79.22279533306506, + "server_steps_per_s": 28.544243577653024, + "server_spec_accept_length": 2.7754385964912283, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 4, + "aggregate_tps": 112.23417563245603, + "server_steps_per_s": 42.43182386817776, + "server_spec_accept_length": 2.6450471698113205, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 8, + "aggregate_tps": 164.1588925662828, + "server_steps_per_s": 60.186578392770954, + "server_spec_accept_length": 2.7275, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 12, + "aggregate_tps": 208.2, + "server_steps_per_s": 73.8, + "server_spec_accept_length": 2.821138211382114, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 16, + "aggregate_tps": 225.31798909844872, + "server_steps_per_s": 79.95154451880438, + "server_spec_accept_length": 2.8181818181818183, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + } + ], + "prefill": { + "8192": { + "ttft_seconds": 3.047, + "tok_per_sec": 2689.0, + "samples": 1, + "method": "integrated_scout" + }, + "32768": { + "ttft_seconds": 11.813, + "tok_per_sec": 2774.0, + "samples": 1, + "method": "integrated_scout" + }, + "65536": { + "ttft_seconds": 23.547, + "tok_per_sec": 2783.0, + "samples": 1, + "method": "integrated_scout" + }, + "131072": { + "ttft_seconds": 47.438, + "tok_per_sec": 2763.0, + "samples": 1, + "method": "scout_only" + } + }, + "column_averages": { + "1": { + "aggregate_tps": 49.41772935173013, + "server_steps_per_s": 18.740041713986507, + "aggregate_tps_vs_public_percent": 5.068301477123849, + "aggregate_tps_vs_preceding_percent": 4.457125434581255, + "server_steps_per_s_vs_public_percent": 5.678414189677716, + "server_steps_per_s_vs_preceding_percent": 0.16201892918366045 + }, + "2": { + "aggregate_tps": 79.33928922705519, + "server_steps_per_s": 29.378749587623084, + "aggregate_tps_vs_public_percent": 3.9101863288222516, + "aggregate_tps_vs_preceding_percent": 2.3643337493709726, + "server_steps_per_s_vs_public_percent": 6.543111702193172, + "server_steps_per_s_vs_preceding_percent": 0.6756998432556394 + }, + "4": { + "aggregate_tps": 118.61380008936804, + "server_steps_per_s": 43.367833952216984, + "aggregate_tps_vs_public_percent": 1.6012560026626543, + "aggregate_tps_vs_preceding_percent": -2.564112482537484, + "server_steps_per_s_vs_public_percent": 2.230693369949188, + "server_steps_per_s_vs_preceding_percent": 0.4911430791546767 + }, + "8": { + "aggregate_tps": 171.7192527370519, + "server_steps_per_s": 61.454428136384905, + "aggregate_tps_vs_public_percent": 3.1921901283812115, + "aggregate_tps_vs_preceding_percent": 1.9193311077898345, + "server_steps_per_s_vs_public_percent": 2.8167028995578214, + "server_steps_per_s_vs_preceding_percent": 0.7080874729887165 + }, + "12": { + "aggregate_tps": 203.22896384180774, + "server_steps_per_s": 72.66236770534239, + "aggregate_tps_vs_public_percent": 4.581166022901795, + "aggregate_tps_vs_preceding_percent": 1.6286209405135255, + "server_steps_per_s_vs_public_percent": 4.981071260009906, + "server_steps_per_s_vs_preceding_percent": 2.301134296779561 + }, + "16": { + "aggregate_tps": 228.98286602968813, + "server_steps_per_s": 81.32841326253032, + "aggregate_tps_vs_public_percent": 1.777172565460261, + "aggregate_tps_vs_preceding_percent": 1.8276218915591613, + "server_steps_per_s_vs_public_percent": 1.811249855611008, + "server_steps_per_s_vs_preceding_percent": 0.41202467903951323 + } + } + }, + { + "configuration": "Loader/RNG + MoE scale sharing", + "image": "sha256:aa221d22f83d90ac497c7e58b2382c5c4d428fb697f0191d6afe971faf67d439", + "source_receipt_name": "moe-sharing-matrix.json", + "source_receipt_sha256": "1d3b93c057fc3bbb7ab2557e78858dc283a06cd9ff3e62ddcedf13f5942f37b7", + "metadata": { + "version": "0.4.32", + "model": "glm-5.3-flash-spark", + "duration_per_test": 20.0, + "temperature": 1.0, + "max_tokens": 2048, + "dcp_size": 4 + }, + "cells": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 49.877309830716065, + "server_steps_per_s": 18.979418098234326, + "server_spec_accept_length": 2.627968337730871, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 1, + "aggregate_tps": 48.925835044218296, + "server_steps_per_s": 18.829185237078896, + "server_spec_accept_length": 2.598404255319149, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 1, + "aggregate_tps": 49.076067905152435, + "server_steps_per_s": 18.879262857390273, + "server_spec_accept_length": 2.59946949602122, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 78.15, + "server_steps_per_s": 29.3, + "server_spec_accept_length": 2.667235494880546, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 119.69575660565239, + "server_steps_per_s": 43.43474779837219, + "server_spec_accept_length": 2.7557603686635943, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 173.8059326283227, + "server_steps_per_s": 61.94067370497355, + "server_spec_accept_length": 2.8060064935064934, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 12, + "aggregate_tps": 199.55687597390622, + "server_steps_per_s": 72.51120398749053, + "server_spec_accept_length": 2.752083333333333, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 16, + "aggregate_tps": 226.29462041078403, + "server_steps_per_s": 82.0330565327409, + "server_spec_accept_length": 2.758578431372549, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 2, + "aggregate_tps": 81.17582252569352, + "server_steps_per_s": 29.145175021563006, + "server_spec_accept_length": 2.7852233676975944, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 4, + "aggregate_tps": 114.37728479252559, + "server_steps_per_s": 42.66613250579326, + "server_spec_accept_length": 2.68075117370892, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 8, + "aggregate_tps": 170.02664521590413, + "server_steps_per_s": 62.33975164628063, + "server_spec_accept_length": 2.72741935483871, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 12, + "aggregate_tps": 200.82131410145885, + "server_steps_per_s": 70.91346153807126, + "server_spec_accept_length": 2.8319209039548023, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 16, + "aggregate_tps": 229.90475230432568, + "server_steps_per_s": 82.2456281807671, + "server_spec_accept_length": 2.795343137254902, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 2, + "aggregate_tps": 80.3786057687884, + "server_steps_per_s": 29.146634615224208, + "server_spec_accept_length": 2.7577319587628866, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 4, + "aggregate_tps": 118.62877762754627, + "server_steps_per_s": 43.702701348213076, + "server_spec_accept_length": 2.7144495412844036, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 8, + "aggregate_tps": 170.4363490778019, + "server_steps_per_s": 61.64931945503581, + "server_spec_accept_length": 2.7646103896103895, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 12, + "aggregate_tps": 201.6112890318451, + "server_steps_per_s": 70.85668534849657, + "server_spec_accept_length": 2.8453389830508478, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 16, + "aggregate_tps": 229.17713252064863, + "server_steps_per_s": 82.78910881061573, + "server_spec_accept_length": 2.7682038834951457, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + } + ], + "prefill": { + "8192": { + "ttft_seconds": 3.063, + "tok_per_sec": 2675.0, + "samples": 1, + "method": "integrated_scout" + }, + "32768": { + "ttft_seconds": 11.86, + "tok_per_sec": 2763.0, + "samples": 1, + "method": "integrated_scout" + }, + "65536": { + "ttft_seconds": 23.609, + "tok_per_sec": 2776.0, + "samples": 1, + "method": "integrated_scout" + }, + "131072": { + "ttft_seconds": 47.532, + "tok_per_sec": 2758.0, + "samples": 1, + "method": "scout_only" + } + }, + "column_averages": { + "1": { + "aggregate_tps": 49.2930709266956, + "server_steps_per_s": 18.895955397567832, + "aggregate_tps_vs_public_percent": 4.803262003335629, + "aggregate_tps_vs_preceding_percent": -0.2522544573978269, + "server_steps_per_s_vs_public_percent": 6.5576390645641025, + "server_steps_per_s_vs_preceding_percent": 0.8319815182959767 + }, + "2": { + "aggregate_tps": 79.90147609816064, + "server_steps_per_s": 29.19726987892907, + "aggregate_tps_vs_public_percent": 4.646479067228393, + "aggregate_tps_vs_preceding_percent": 0.7085857165881171, + "server_steps_per_s_vs_public_percent": 5.884968889906372, + "server_steps_per_s_vs_preceding_percent": -0.6177244138752203 + }, + "4": { + "aggregate_tps": 117.56727300857474, + "server_steps_per_s": 43.26786055079284, + "aggregate_tps_vs_public_percent": 0.7048302430184172, + "aggregate_tps_vs_preceding_percent": -0.882297911376928, + "server_steps_per_s_vs_public_percent": 1.9950267660463261, + "server_steps_per_s_vs_preceding_percent": -0.23052431332930556 + }, + "8": { + "aggregate_tps": 171.42297564067624, + "server_steps_per_s": 61.976581602096665, + "aggregate_tps_vs_public_percent": 3.0141467117428933, + "aggregate_tps_vs_preceding_percent": -0.17253574753748335, + "server_steps_per_s_vs_public_percent": 3.6902949153018394, + "server_steps_per_s_vs_preceding_percent": 0.8496596283557478 + }, + "12": { + "aggregate_tps": 200.6631597024034, + "server_steps_per_s": 71.42711695801945, + "aggregate_tps_vs_public_percent": 3.2608090048235727, + "aggregate_tps_vs_preceding_percent": -1.262518929832046, + "server_steps_per_s_vs_public_percent": 3.196406779290606, + "server_steps_per_s_vs_preceding_percent": -1.6999869207841911 + }, + "16": { + "aggregate_tps": 228.4588350785861, + "server_steps_per_s": 82.3559311747079, + "aggregate_tps_vs_public_percent": 1.5442538782907356, + "aggregate_tps_vs_preceding_percent": -0.22885159933061505, + "server_steps_per_s_vs_public_percent": 3.097551637377416, + "server_steps_per_s_vs_preceding_percent": 1.2634181228406982 + } + } + }, + { + "configuration": "MoE scale sharing + top-k selector", + "image": "sha256:3b4768e5ba31cadcc882dffa06d7b667af44abdf157d5c11b7ac7fe962e80c43", + "source_receipt_name": "topk512-matrix.json", + "source_receipt_sha256": "fe5c17bd0121b92522fa862ed2ef33b59f90efd8675864c1438dcf31f571a90b", + "metadata": { + "version": "0.4.32", + "model": "glm-5.3-flash-spark", + "duration_per_test": 20.0, + "temperature": 1.0, + "max_tokens": 2048, + "dcp_size": 4 + }, + "cells": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 52.6, + "server_steps_per_s": 18.85, + "server_spec_accept_length": 2.790450928381963, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 1, + "aggregate_tps": 46.2, + "server_steps_per_s": 17.958466453674124, + "server_spec_accept_length": 2.5769230769230766, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 1, + "aggregate_tps": 50.05, + "server_steps_per_s": 18.75, + "server_spec_accept_length": 2.6693333333333333, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 82.47784065284291, + "server_steps_per_s": 29.145175021223178, + "server_spec_accept_length": 2.8298969072164946, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 121.63853973634211, + "server_steps_per_s": 43.4852769732887, + "server_spec_accept_length": 2.7972350230414746, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 170.11367619974143, + "server_steps_per_s": 60.49376533685241, + "server_spec_accept_length": 2.812086092715232, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 12, + "aggregate_tps": 198.77522337094305, + "server_steps_per_s": 71.07720108415539, + "server_spec_accept_length": 2.7966101694915255, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 8192, + "concurrency": 16, + "aggregate_tps": 233.02236934401157, + "server_steps_per_s": 82.65623432607212, + "server_spec_accept_length": 2.8191747572815533, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 2, + "aggregate_tps": 81.02558966397989, + "server_steps_per_s": 29.34548550252918, + "server_spec_accept_length": 2.7610921501706485, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 4, + "aggregate_tps": 121.54115586690078, + "server_steps_per_s": 43.832874655992214, + "server_spec_accept_length": 2.7728310502283104, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 8, + "aggregate_tps": 170.027083960276, + "server_steps_per_s": 63.396529240645684, + "server_spec_accept_length": 2.681962025316456, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 12, + "aggregate_tps": 199.21874999999926, + "server_steps_per_s": 70.91346153846128, + "server_spec_accept_length": 2.8093220338983054, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 32768, + "concurrency": 16, + "aggregate_tps": 241.1358173076914, + "server_steps_per_s": 84.13461538461507, + "server_spec_accept_length": 2.866071428571429, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 2, + "aggregate_tps": 78.37147578747127, + "server_steps_per_s": 29.145175021283244, + "server_spec_accept_length": 2.689003436426117, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 4, + "aggregate_tps": 119.18473634133011, + "server_steps_per_s": 44.06830587410525, + "server_spec_accept_length": 2.7045454545454546, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 8, + "aggregate_tps": 181.63152886974765, + "server_steps_per_s": 64.49997496118968, + "server_spec_accept_length": 2.815993788819876, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 12, + "aggregate_tps": 188.9440184433421, + "server_steps_per_s": 69.16253195008278, + "server_spec_accept_length": 2.7318840579710146, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + }, + { + "context_tokens": 65536, + "concurrency": 16, + "aggregate_tps": 230.29816283505642, + "server_steps_per_s": 82.72261821102288, + "server_spec_accept_length": 2.783980582524272, + "num_errors": 0, + "underfilled": false, + "warmup_timed_out": false, + "capacity_limited": false + } + ], + "prefill": { + "8192": { + "ttft_seconds": 3.094, + "tok_per_sec": 2648.0, + "samples": 1, + "method": "integrated_scout" + }, + "32768": { + "ttft_seconds": 11.859, + "tok_per_sec": 2763.0, + "samples": 1, + "method": "integrated_scout" + }, + "65536": { + "ttft_seconds": 23.532, + "tok_per_sec": 2785.0, + "samples": 1, + "method": "integrated_scout" + }, + "131072": { + "ttft_seconds": 47.422, + "tok_per_sec": 2764.0, + "samples": 1, + "method": "scout_only" + } + }, + "column_averages": { + "1": { + "aggregate_tps": 49.61666666666667, + "server_steps_per_s": 18.519488817891375, + "aggregate_tps_vs_public_percent": 5.491267203291317, + "aggregate_tps_vs_preceding_percent": 0.656473078036246, + "server_steps_per_s_vs_public_percent": 4.434677347465721, + "server_steps_per_s_vs_preceding_percent": -1.9923130201975003 + }, + "2": { + "aggregate_tps": 80.62496870143136, + "server_steps_per_s": 29.211945181678534, + "aggregate_tps_vs_public_percent": 5.594032945587912, + "aggregate_tps_vs_preceding_percent": 0.9054808979772622, + "server_steps_per_s_vs_public_percent": 5.938189412973083, + "server_steps_per_s_vs_preceding_percent": 0.05026258554419183 + }, + "4": { + "aggregate_tps": 120.78814398152433, + "server_steps_per_s": 43.795485834462056, + "aggregate_tps_vs_public_percent": 3.46373802632558, + "aggregate_tps_vs_preceding_percent": 2.7395982661898355, + "server_steps_per_s_vs_public_percent": 3.238794177820936, + "server_steps_per_s_vs_preceding_percent": 1.2194392719044433 + }, + "8": { + "aggregate_tps": 173.92409634325503, + "server_steps_per_s": 62.796756512895925, + "aggregate_tps_vs_public_percent": 4.517158860705339, + "aggregate_tps_vs_preceding_percent": 1.4590347024552086, + "server_steps_per_s_vs_public_percent": 5.062493513296618, + "server_steps_per_s_vs_preceding_percent": 1.3233626147130195 + }, + "12": { + "aggregate_tps": 195.64599727142814, + "server_steps_per_s": 70.38439819089982, + "aggregate_tps_vs_public_percent": 0.6789885436115917, + "aggregate_tps_vs_preceding_percent": -2.500290755122181, + "server_steps_per_s_vs_public_percent": 1.6899084824137711, + "server_steps_per_s_vs_preceding_percent": -1.4598360000061095 + }, + "16": { + "aggregate_tps": 234.81878316225314, + "server_steps_per_s": 83.17115597390335, + "aggregate_tps_vs_public_percent": 4.371092169042257, + "aggregate_tps_vs_preceding_percent": 2.7838486007684127, + "server_steps_per_s_vs_public_percent": 4.118093565958447, + "server_steps_per_s_vs_preceding_percent": 0.989879887905154 + } + } + } + ], + "conditions": "Four DGX Sparks, GLM-5.3-Flash-NVFP4-Spark, native MTP3, TP4/DCP4, SIRCL/mesh and SparkCache.", + "interpretation": "Descriptive matrices from separate runs, not a matched isolated transport comparison or repeated-run confidence interval.", + "topk_run_continuity": "Eight cells preceded benchmark-client power loss; ten resumed with the exact same 0.4.32 harness. Four server containers stayed running.", + "topk_host_state": "Top-k startup followed a host reboot and the memory startup gate; reboot effects are not isolated.", + "transport": "Measured compute images retain transport bundle4204fabc. These runs do not qualify the combined stream-safety bundle69313e19 image.", + "image_identity_source": "Deployment receipts associate the four compute images with runs; the historical reference is identified by its hashed benchmark receipt only." +} diff --git a/performance/records/glm53-flash/spark-mtp3-compute-matrices-20260905.md b/performance/records/glm53-flash/spark-mtp3-compute-matrices-20260905.md new file mode 100644 index 00000000..428831a5 --- /dev/null +++ b/performance/records/glm53-flash/spark-mtp3-compute-matrices-20260905.md @@ -0,0 +1,62 @@ +# GLM native-MTP3 compute comparison + +Status: research-only measurements. All five matrices contain 18 completed +decode cells with no request errors, underfilled cells, capacity-limited cells, +or warm-up timeouts. + +## Conditions + +Four DGX Sparks serve GLM-5.3-Flash-NVFP4-Spark with TP4/DCP4, native MTP3, +SIRCL/hardware-forwarded mesh, and SparkCache. The harness version is 0.4.32; +temperature is 1, the maximum output is 2,048 tokens, and each decode cell has +a 20-second measurement window. Contexts are 8K, 32K, and 64K tokens. Each +column below is their equally weighted arithmetic mean, not per-request +throughput and not a mean of repeated benchmark runs. + +The [sanitized matrices](spark-mtp3-compute-matrices-20260905.json) retain all +cell values, acceptance lengths, prefill scouts, configuration labels, full +compute-image identities, and hashes of the original receipts. The historical +mesh reference is identified by its benchmark receipt; its filename contains +a DFlash label but the associated serving configuration used native MTP3. + +## Aggregate decode tokens per second + +The compute additions are cumulative within the four compute-image rows. + +| Configuration | C1 | C2 | C4 | C8 | C12 | C16 | +|---|---:|---:|---:|---:|---:|---:| +| Mesh MTP3 reference | 47.0 | 76.4 | 116.7 | 166.4 | 194.3 | 225.0 | +| NVFP4 MTP proposal head | 47.3 | 77.5 | 121.7 | 168.5 | 200.0 | 224.9 | +| Proposal head + loader/RNG fixes | 49.4 | 79.3 | 118.6 | 171.7 | 203.2 | 229.0 | +| Loader/RNG + MoE scale sharing | 49.3 | 79.9 | 117.6 | 171.4 | 200.7 | 228.5 | +| MoE scale sharing + top-k selector | 49.6 | 80.6 | 120.8 | 173.9 | 195.6 | 234.8 | + +## MTP-normalized aggregate sequence steps per second + +Normalization divides output throughput by observed accepted length. These +are aggregate sequence-step rates, not batched engine iterations per second. + +| Configuration | C1 | C2 | C4 | C8 | C12 | C16 | +|---|---:|---:|---:|---:|---:|---:| +| Mesh MTP3 reference | 17.7 | 27.6 | 42.4 | 59.8 | 69.2 | 79.9 | +| NVFP4 MTP proposal head | 18.7 | 29.2 | 43.2 | 61.0 | 71.0 | 81.0 | +| Proposal head + loader/RNG fixes | 18.7 | 29.4 | 43.4 | 61.5 | 72.7 | 81.3 | +| Loader/RNG + MoE scale sharing | 18.9 | 29.2 | 43.3 | 62.0 | 71.4 | 82.4 | +| MoE scale sharing + top-k selector | 18.5 | 29.2 | 43.8 | 62.8 | 70.4 | 83.2 | + +The selector image's C4/C8/C16 normalized means exceed the scale-sharing +image's means, while C1/C12 are lower. This single matrix does not establish +repeatability of those differences. The independent-RNG correction changes +sampling behavior; raw token rates alone must not be interpreted as compute +speed when acceptance differs. + +## Scope + +The top-k matrix resumed ten cells after the benchmark client lost power; +the serving containers stayed up and the resumed harness matched the recorded +source hash. Its initial deployment followed a host reboot to restore memory +contiguity. The comparison does not isolate reboot effects. + +These compute images use transport bundle `4204fabc`. They are not performance +qualification of the combined image containing stream-safety bundle +`69313e19`; that image requires its own deployment and validation receipts. diff --git a/runtime/glm53-spark-mtp3-mesh/Dockerfile b/runtime/glm53-spark-mtp3-mesh/Dockerfile index 5e97016b..86053fa1 100644 --- a/runtime/glm53-spark-mtp3-mesh/Dockerfile +++ b/runtime/glm53-spark-mtp3-mesh/Dockerfile @@ -40,7 +40,7 @@ LABEL org.opencontainers.image.title="SparkRing GLM-5.3 Spark MTP3 mesh runtime" org.sparkring.mesh.source-receipt-sha256="${SOURCE_RECEIPT_SHA256}" \ org.sparkring.mesh.default-speculation="mtp3" \ org.sparkring.compute.source-lock-sha256="${COMPUTE_SOURCE_LOCK_SHA256}" \ - org.sparkring.b12x.composition="b58f34eaf978277621efced6678e6713fd7122e4" \ + org.sparkring.b12x.composition="ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301+pr316-9ac14282" \ org.sparkring.vllm.compute-source="${COMPUTE_SOURCE_LOCK_SHA256}" \ org.sparkring.mesh.proposal-head="nvfp4-a16" \ org.sparkring.sircl.manifest-sha256="${BUNDLE_MANIFEST_SHA256}" diff --git a/runtime/glm53-spark-mtp3-mesh/README.md b/runtime/glm53-spark-mtp3-mesh/README.md index 88c1efb2..d32ea8ce 100644 --- a/runtime/glm53-spark-mtp3-mesh/README.md +++ b/runtime/glm53-spark-mtp3-mesh/README.md @@ -36,6 +36,12 @@ their limitations. ## Operator benchmark observations +The [compute matrices](../../performance/records/glm53-flash/spark-mtp3-compute-matrices-20260905.md) +record full C1/C2/C4/C8/C12/C16 results for the proposal head, loader/RNG fixes, +shared MoE scales, and top-k selector. Column averages give each of the +8K/32K/64K contexts equal weight. The records identify their measured images; +they do not substitute for validation of the combined stream-safety image. + The [consolidated validation report](../../performance/records/glm53-flash/spark-mtp3-validation-summary-20260905.md) collects the completed tests, three-pass prefill measurements, and remaining work. Use that report to avoid repeating checks already covered by receipts. diff --git a/runtime/glm53-spark-mtp3-mesh/compute/README.md b/runtime/glm53-spark-mtp3-mesh/compute/README.md index ba35aa58..c015651a 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/README.md +++ b/runtime/glm53-spark-mtp3-mesh/compute/README.md @@ -7,10 +7,10 @@ the profile's validation records, not to this source-preparation tooling. `source-lock.json` is the authoritative input. It binds: - the vLLM source revision already present in the parent image; -- a reviewable vLLM patch, a byte-exact 14-file replacement archive, and every +- a reviewable vLLM patch, a byte-exact 24-file replacement archive, and every base and resulting file hash; -- B12X revision `b58f34eaf978277621efced6678e6713fd7122e4`, its Git tree, - source archive, and all 385 installed package-file hashes; +- B12X revision `ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301`, its Git tree, + source archive, three source-checked selector overrides, and all 385 installed package-file hashes; - seven NVIDIA CUDA 13.3 SBSA redistributable archives; and - the five environment settings that select metadata reuse, dense-kernel policy, and the NVFP4 native-MTP proposal head. @@ -21,6 +21,15 @@ uses BF16 activations. `VLLM_MXFP8_LM_HEAD=0` leaves the target/verifier head unchanged. Rejection sampling therefore retains the target model's sampling contract, while proposal-head quantization can change acceptance length. +Deferred GLM weights and scales own their storage when a loader reuses input +buffers. Draft proposal randomness is independent of rejection-sampling +randomness. These changes derive from vLLM revision `17e341b9` and +[PR 653](https://github.com/local-inference-lab/vllm/pull/653), respectively. +B12X shares native MoE scale storage and includes the top-k-512 selector from +[PR 316](https://github.com/local-inference-lab/b12x/pull/316). The source lock +records full donor identities. The selector archive is applied only after all +base and resulting file hashes pass verification. + The image builder calls `prepare_compute_source.prepare(destination, cache)` while network access is available. The prepared directory contains the pinned B12X source and CUDA archives. Docker copies that directory into the build and @@ -40,7 +49,7 @@ from revision `a8c796f3af74106b2d8d441e9ec54588936a5388`; vLLM is licensed under Apache License 2.0. B12X source archives use LF endings. The tested ARM64 image was assembled from -a Windows checkout and contains CRLF bytes for Python and C source files. The +a Windows checkout and contains CRLF bytes for Python, C, and package Markdown files. The preparation step performs that deterministic byte conversion so the public -image can be compared exactly with the tested image. Markdown and compressed -profile data retain the archive bytes. +image can be compared exactly with the tested image. Compressed profile data +retain the archive bytes. diff --git a/runtime/glm53-spark-mtp3-mesh/compute/b12x-selector-files.tar.gz b/runtime/glm53-spark-mtp3-mesh/compute/b12x-selector-files.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1c76ea5d24f08a533c07ae9f4fc48af929bf9519 GIT binary patch literal 54422 zcmV)7K*zryiwFP!00002|Lnc(a@$CfD7wEpqTc}{$83`pNr{wNt&vT~k!`u#;Scp# za`)_Y_(C8FN@#%u4S_P_ld{w%XRNTK_G`|tT@V`Jm}Jeo$~#C!4n#v@N zS#&=4T3Hx+i#V9K54^xjlk4s{Nfu!m4Iecx+cu;GLb#K2I0>XJ_7(T@$>Cmn}!#Jc>hx5X|B4?#Y{@ zH(t7&&!hRdm^}3O^;h{t2;E}qi4Tu2q3<-9M4!FZI0@1$?Cy6*QQp0P=_coCF!MIO z(M=w9({LKiL+C!3PjJc^3^6?qzy1nPnlICN&wHJmM|tMW!z);fl*V@+C!-)1lkxuR zfBtX4WN?Y->;g`_>mf6+xDTvr&9QvtXkPl+JF5O(yrI~5r27o7(Airk;e}P@Zj0;+tZidzdwF|;=g%)=rOKjgB@sq zxc9#854O!Sqh zR^ueb;o-AhFdECIZ-T`Fdg*v4A@GCwI26q%K_1ZZ14|H1)g#el3Y3*6Nvs;g>Czv= z)(Wvx@hr)-ALC`t-r$ghpc>^#I=+xIWN30U zF97lR&Yh@&vYeho6gW2*XDZKK~T&fD*DK^#w$K8jN#z(=K%h15cxQ7LC;u znIHR$JO#q>mugzYH*$ZQqdeiKPnXkam};XGt^E6amU0{w2?>fpamFFJ*ZPU|4tDe6|;9R@0 zCP@dd|K{l3_ecKW`_to2`SG(Kj$b}II&r-@KK}9O-QnrWpN<@FPmbQbJbZ0q!9BR* zc&5#54bp#i`26LcI{FX)^oMsxCqDr9Zq`A;IBeK(3cU5-zkPT7`nBxgD zrH==YKVJAjb~Be7#DDqr`O%;J9{^ka^OtYr(;p9!TweYkNBV>3{vWyF>C4yF3u`5; zm&dP;-txQe-yOgI(R};!;kzG8AL7IEB32%54Vw6$;>k?5!?TTXC()9Dpc16={d_s| zKlykz6A%R-HpL9!!zn(Uf`H}s2m2%W4%TWp2K*-RayAE`5ip|VtaytIS@s6>CP5hl z0`!sk#K2*#_eC!aDMl=MkSDWfjF5bm6#Yn%b@|;Ajuw75odutf*2K?qm>u%xAOk$* zs#UDM491_95ggtH^y1z4+`rmIzFvAWA8UOWk6o<|AYy-&XJx%eL7V~cOKXsUq~Ko% zQ7)UDo5P+3fXmPRY=6H0c^5FhR}4h#L1Q3HT(eo=&-#1w{mY#J&dCpVXFIaPTzf#6>0IA{<}h zKgSor{2b3iHE!VW$M25(qqm1oUxTta1Zk@m&O!VHt_FnH+HlTxqvLJ#H`+a5$YI)Q zlR^Xi{)I8QvArR1x&hoe3)QcAqW+2h_$%VLbaD*6{`O!OXZR!NQTXIlmzBf0>51K|tjjv$1jX5Nq*ug!jxzHDb2_Mz z5;?LOFiEcGJVVsqNy$Sk8flWW=!msH-W;eHzWT|3efS^8?@xV8yBd1i`dd0ZaN&4r z?-CToXJHYee&ofpdf}%yoa5^WY=#joY6z+;(o*0_d29}cNC$z! z2Dp0t`VFr3;sW*x(D)5*1#HecllEE&$|EaufEiemv4oVuaDYq0;t(awfH5t}xG)2O zg4Fdmn!(C>&vv~fsw}KNqrynyWfzmG<@c}O zbieKGbYFmOZ~@yFwmBe|pAHzlYFy~*gYDh@9>RJ(=uYD$Xw4Jv`N>baFXV7%K+c)e z5_zd5eLn5)AqT?Y1LcfB^P{CW{Zp4!@RK+F!8Y~v{nP&T7XIv0Uq7PhBaQ(VJ3qGt&GXY=&pMJ~5Kg%l;mY z0T2Tqx@u8^X)g5t{5t8z(05GaIU(y1nFzwkfg<*Bk)aVP^ZH$e*Sdz8fIbXE4RUY) zRR?H)7R`|%!|$EJt8Ra%4^PA1dCyDx{{w${yM37JRp$L+8~&!=+TLiHBfL1l6{9Y8 z23?>5+Er{MYo&X;450t51kvB#-H|{#9yZvA*W26h4#9o5-}bs)n%@999PDUdb?2~< z3pP6u%9A;or7|+!5Fbp&p#Qgfb%J|4G0T723*kSEcG#oXA?cED#fpkZ&9*aNXg;G4S1juwJ& z1Y{+LO%^o-{SN8n$xs_P*^~Yprb!pK9MEnH*cy@no-PPDjez2S)JU4ZH#1`5vmm=f zD;PqhBYl2z4eW1pql5|=L-!hvz)!%lQaZ zHcgMWHM2q<1PBs1N+3b^guE7PYZhDsZ{K_Ml;;5F2C&bJEQP2Hl2n!iAXAND48U>Z z`JIJ?hPj|{>MT#Zxz8wpGZvU7JV0Tw?TKxKUC2RZC>8x}U%@;HW}(&YH{Ibm8IQi_&P|%hGfo`U8bbf)RMDGH}2}A^9*-fv4 zUmDsjPSRqRIJC^cFR?waOEf{+ER#S*lXW>`{@MIOVYkdD71PK+{}c0yMw{1IhYtd^ z9TzVs_O`-Y!*kNH?Em0X^1mG|{L`EpO=wHluw)e2%OV=G3m? zGh$ZAqGdP2Qvw}KqFHyFxh*$&L`pEpH@Ya*@%DFj_wg|UeIOV*DkH)v$_A0v-tI#f zsYlZSRKw3*I-f%HA!Ez7oH-(eR)nm1=AhRgcMzO-0DMrg<*NHcxU%U$-Sor)9H5Rg z0pYt#Q+_3^6CoX#0iF`V#)mdU;Xfv~HqKVszq-;Q$qG6f2YTMSkPJJSaQO0xtdWWs zNfs^w)TQa;?i2Qulg$&b1Daq9l?!xbkAgIf!nDl_E~p!Vg@AIIq|u)V@31$#?|BLy zA~7U)9*VI#g|rYq4@-35Z3?qsa!pINiLf8`_u7xjG~>N_e{$-LLiUD+fEK14ar7rs z2RXBW#6cr~(J+m};ED}FEY~oNkVJyBXIX&_g46|wCP;$po=3nMc_PWSNfcjZB<_2# z_!HEBC-QR^2FMIU94-Ylie(-l%FBVu<$Rf=>y;3@KFhXwCURg{w0mUT0%GwM&;^YDb3-Ox`uv30RD#&3%2#W#naS2nPj<~ zd5zbiganf>T6l1JR$6piiMe!YSEXWhw$;5Q$KJ6f-oC{VXu5O^*@|IQodWlEM*M5< zuPhx)IxAz8NgbdtL>tzo>inSz^q+`$U0c(nW{IIMM)4|irUe-Kb2uNKrc0D`*t+Y# zK+)ti#e_VgKrKNs5kEct1v?mrScyM4dEZ&llI(RtSzsI| za}gP4`jAIhHYm$&S=2FQ;XJ<51XqYFL zAl^S>=*kZUKch)k3R7gu2PE0z1v;LIVt~Zq-z|p|tK&ILS7o9S0D#6@G%#Fi1zat* zeA2dw6+e&W+TK@9I~v(3&{p3PzWEpw9MMm^AeC&EQEVVm1ci&C1i_@?FH&4|^dhwQ zNxMfRi7!A5=ZLFidGCn9pEJ^4Y# zptsz3SsaYoo@wiRW_*J&q=LhtrUN_&z;Mu~M+=8`%uSHx$p;AdLzf~N&dC6Y@LYX* z7e#*$Bi*`VhkSf7E{G3ySHG6Qy7(g9FKNmW8 zPEHR`kNlsH-#yo&c7Q5&Q9~B9^2KC2VsgcA0_Rf zh+?|aC2HB8_>u^MBRN#$v}{H3wm=H64^K{!---xAV9iJef9QWZK%ggRPsi^z*j0;& z+9<|+AhAjD3c>tF`QUNg6n@n9Ly$UMh`8}QxU5)Bz`55#-8c^Ew_phz5dJ`(M$mgM zz4jwqP37T4Ro8!U{O;M2nkcHl-s8vK&NhX8vO_MDu?wv4bIeN-PlE^P-fou;LDs%; zqZ05}xS75`Lf_~PZ-gHUW@Bv$W*{vJM8r7$Ne27CravM%rhatDGhPx!5a+zAznj%3@;dumv9ILJPydE>lGNaGf~b%vLj^_F(CN?ZXXg|6(f}VQ*agvwa7&O z(L`?A4o}hGupUnp*6lii?L|H$c22fr>^tgiU1F^)v||7RnZq>NYBOv&j3rQYw!Gg?MoBzYhRtOnlT+dX(a_xKJyZhv}ws<$?QY+^{%1Mir3M2mL-FRVt%Y)=g)zXan@klj9;kEK-* zmYU4750JMZolQev%3V+3Q`_muapH`1_U{CBNvC4|pJ67vw zua6Gjl4Mfx!7fGt9UuRwOElH3_rLxMFOFa=t!om$$16WTag54V8L(=g;5Y~wrKpEi zn7Z4(#Ofv;3eqe=d_!}E5MvQPNTsBInR>7Mrm(x%18>9J5zy*kEb_EP|FV2Kwd1me zV;f5eUZ-}t46og;8z8q8XF^Nb*c%L$1XcCRwo-`4$|xsPcOB25$bPS#NAeghPD#?D z#qnaDX^#Y2vO+MY@${5-EX8fW8B(ruHHuj&Lz9ETrJ@&2G1_1_R6ao6&{r8t-O>gk zd+E1z%ac3G+Wf9Xvs1=908ph^(xL^ZA>(ZYX_so5YNrj&O(4Qi;e+ZN&B?Rcl2CA5 z6p$Z`Vw+Mvq2;b1j3h2dhOGAUNi9wMNRcNKl)7>gAY4_r{g0szW|>-LKUMPKyeVqAttncoR(sX5~w2waQsL;qvjax?4w zSl@j3#1>6-G+V0?Rj`mtc0ZFHvL#{x1S;v*w)wMf{v1>>IKavY**d%8GAy~L0Hv?I zl&-*1EZx}|{f##yw?WJ$3=i7c24ib54@4o9zg3&0uOmqRqF?pJpz4c(firyLMj8tJ zz7Xnk?*glP4%Q|vZdm+{pP~eZT=hT&AE;3955R^$>doq15}JSda57-D_dd~@gQtX=q}CHEBmYE7g>sXE0tRJDdePL^cx! zGTzpq^w)%ZR-y{jdC|VHR}m zJ=}9X=~j2yd92zr*n6lKHE5Rk<87aeJDF<`q+_xz&T=}2H$F!69tad@j~5t>|Jiy$ z>L9E^G|g9xi11XwImGNM-e&Y-+cgiq7yw+{zcjxQQkHLsZMrkyyxyIP;m78a>mXg| zN4>%`5`nN(xQB?>1IbhBK+q_G^ReR=4bTrGboY{JhqZZ!Tqwm$mJg*%=ZG$T6-ER$ zm%^&z^&m1Ro}CqI4D6ScZhnaLaCRnFst5!pO`XTb7M>u7FEolEj}J9t){nHyU|RAN z=t%1_AI7XS2|d3hOcT@w&SxeA}izPY=actJwnqSUc#!AtPCUedPH-r$Ac2w4sNGCSgD6JX1}#)Fl*)FfMM`wbKO6r22;@MW zK@>a{31;ZH58V>k2wEE3`4D|9&cA#6x0 z?0EJp)dI6p_t+SXeh7~h#>H;?kbnMYuHx8x;!+7Kj&QR9&1Wsqsn|m-JOPHsCQKuO zT8-H&^j5KnE)2=dv;$NO!8k;u(tmEEQyvQynzGuhbCwbMHcIra%{OWs=Hmh*)(f@# z_M)gG(t3kuob|}$E+t6vBr8XvG}psxM5#I8JGuTl3R8bCDd}HTzHvz{lt?L>!$G=V zC%P2&uQ<7;g9WY1N?;K;V&KRQ#yMu}IKX^72WPg-eZ~R8g&U*LC()ZwI`dZWzd8<5 zPvXDs({$BA#v^Bl+F{!M5(nCEXV5?R45~SSw2rlN#yCUx*sxCu?*(R4_%g?WZ;K6t`hA%&;G%6GugZW56C8VSY=ymy~i1%Tg?<&e7w0k@BExB6bFuF2z zV4fD~tHq2ks&SlS)gWGr-s zo_lz!UJT?5bpB_9;)V3QWEgOe^@|tsKwzSPcfzAUkydE!z^F<a6twwj1U*Ko*OMR9XbLBSKb&6Z7z-^^-%Hw2PBi&!%M`t8%3L_a=|RoCN>fRg+;R z053jd7&jrtL}-xz9%%pGQUA8jYgz=^>|5>Md)mLhEB;M?eKdGJ^0bmeOuReK%dc=D ze!+#1teD~!e3kJO=-%|6wRpB4e)))56||E9%@gV+23>pvgzN7Qoa!A9*xG7p)qZI^ zKcbl!eU->+k0@%FsLSwH8U6Yg(p(muiZ-1p@*7R#o0i_rL4)m5uQc%bPS5j3ePXlf zE~mH=E!+OI?|3$FJliQhle_7$G5~29odPeM!akk?s;5=F&=S+D8T4v2!>$*ICT zj#+Y<+{r4}Wsn8Pb-CoOyLB$uk4_&bzqKl^|u4pw;y3;oq$kz zm05WaqduXukoK%&g(l_J>KsGa(S}kUw=*fDquu%nayjVGh4Xfajkq*Ec?bk5x?@xs zxrRcALhHzT;sv@txkyEV44GjFscjzY528P6~XMDRvLjX_=F58`&QSccm|M zj9TqsC5+TGpJeuv@+y_{UC?+RYvVl7b1;-`1N6LK@qAG6e8>G9x1c@#@Z&U6Tp00nPbAHNfa`QSs4MEE0R#q2dh)B&;?ANlMeWNgVn!O z+qf>fO6Fg?HaOlVzcBa7iachr0ls3*K;QjkdEYK6P_xao9eP4HP4`891ur*VObs___q;f!a2lqV(jD^;``7r8(gib%mV_dH5nOo*RUL%Vr2X%5uS@Y9NFJ9`yw(Eh#Qxd# zw+)`dl^|LrHfJW$($-7zv&;TnAsH%4F~0siLERa&wXxn8)ZKl3tb==ky0@#3^}e9) zZ|h^-xhJT9*we>)Ur_t~V!3EPB5}#^o$cPX{;KbM_31oaNV-=kvvxH%l&CIPsoB^k zhH;^2Ja!LAyzJM#9MrwsseJi~lgJ7|%RXk65@d=OG|7jK`a$KST~0p|B8tjw)ga1| zRcmL^u9Bddr~R@f?Qp%gGk>{N9^#WyqZz>TaayQ5GK=gq%!;OPRPDT>l>|8}fMX%R zv<|!}N@Z@!l3le1Nozotn;lmD(IpZi@^aTZq($V;<~ul}Ydh-KcGR!wsJ~uEgW8S; zH60D?j0)J_d3>)ixaZvv|~fz_Mf>ebX7a*!KxhmP}c-~D*te!Sy)?1b!u>xAlp z2`F+tbwO1@jNiTB|7+i-k-b}(oldq(k8yn_{nBG(p_4)BF)r0aTT5i#+eQ3n6)z+% z0LO*-VPBL>-Bzr1UmNzd&i_Gb`#?=~OVV}U(ZaAbEp!;N09cEt$2bE}f4ttV`fx#> zOzrD|E4;Ou_+np8NmX2}pV6&=r8_cY2f8V#i@7H~=f@_7NihJw` zP+E#Xqsq20i_J+oX>kUa14oKAcb-F0huKKoR*})`AtuPE7z0N(uN*^%O5D2AEbVp2;))HCr#(BV?+MEn+<&?xTD+%928~|55hJWuDvoj}$MUt~cFjh!* zrEyfZ8)Ys>De{T=ir)|Ag-J)ENAEvT9f$wE#S~FP`b$RF7np+cNi+w|3!7-ChmW@Z8^v%z{?;z5Heo;w@eVuZ`k?_w2{_RLYw&&B=0@2Uz=g zIm6<}&rT1wM8dWt%`n|oCjEBWLSkA$Tp)?)HpVJZa@tA) zlh(5>^Kx2Q3|DSLJ@P z-9bA|=a!a6vS+zoxm(1{U;5fO{;H;0VC9KtmATR*?kspcyqaNG#oAOT)b@s^_cX=U zbsd(Z26av9lQk^!2D~%XFQ~@XK3=ym6aQjWH984i^7uqu=4Zh|+O4aZp@Xa~iZikR z3-XpWOxid8A^{;jnn$_sx3Vyv5`|lg6IKgp7GL+6d5C(ERxM%smPrr+LH&d$*HM}# zu0Xx%2A51r}hi6lk(0{RE1o99vbb_*eVxu=l!fymvnuDn^RcXphh73{eYVX> zB_n1O-X!x$PnF$_vI2FZaLR?n#xTUa-M^Zm?ylFu0;pUmb%-T} zjLNFwIhq(bF@>YvsSd9weX0@)XLt#A8hr*ViYhU%1gTb()vU=|7Nkp|^frT_*->Iw z%zDEaUfX}Q3>F$r6o0I!np!f>+UO@TT?_y4k&5`#rdA%2tJbK&u1lZTZQWX+NCWOm zVWa^!FBtV*S#u$fBWD)ofeS$uloU=X1l~LhC!9N%_rBm#V)%T5szuQ#!`>d6oxgTB znC;f1Eytd%rKn0WYDO%pX`_l1N@=uBJ;9;{*GZ0KLxs%rhc3alyyW*=1mJ;pifOj7 zf;H#DrtC1+!Br@ZZac-I_cu8LU}02N62osLFk<|e1j<}cT+7vSC}1-4B8fAn-!9TBfLu@n zTFcsLB7wb@H-f{uxndjTcjhd>G7ut|Q{d;Lq7*sbvL(q#t{7E0Pv#&K;c3>gYLcsB z(N2 z5Ci$Qmeaik_3*tEmP9IgUeBI)V&{o0rl_Z8w|78Mt64RjV}_k9LWuF z=q~vnItV|H;T+7YiVCh}ek5vEBc)%XI*;XR>S_qotiQmTMFcPttW$s&(^0pT7Z9kH zgDS5e5~9icCM#IiS=%8+Yv__Hu{h7D8pE?6j$b}II??H=-h46LSk8r1E-On4=zr0E z{4V|LBS80p^#f!x$<*7hA^pWM65Y1ZUbzo&&+B6t4vpEJ9KCya_`2a{v5;kf!i?8P z?+#C2{&ZA^w{6;hn;kKZx;2hzTE%NvaWEwtqkau2MoJp$lnlnE^vY78e+~4zT-;iI zT}Oyqi)>nWAAlJym63MzYq;3lJ+ViY~^h>lJX{ya z0?TxS;#5pwv}kI%3V`LQfIXWFQ#wV;7O8P-!J8$=q

RB*QD!2nC6C9eK7&djb&J z4&z911l^kjza**f;fN>E9G;lL)E7Y(T#3^B zC5_a_%AMSh3zTs(EI-njgR&~Z2uQ7qT$_V5zj&8snw3Xf{F-oKF;-)e`O#dbnxE@(AP5N~7vZTxrR+R%eWn4b;pDq|gcq9!L1p9!hU+!EF$Dg7P!`uJNb#>m`xJ>a9^d_wxU z6(fRMt!1`$R<9#=oTpKOIh^n!PB%E8CvYH*53sTwMZm{5ZLiKpw4%aig`*FPmdR3| z=<};{2nzmWIWRRJpW>r_yKFbFH3*qmovv;bV3KN z59fzJK@WGSBmKWh){&7J^vTDnu6dc3*>j58(7xD-qRA?`>d4hEC2BeXc z#W9j4RB6G@%@Kq3;w7S`BxIl$Td;Hg_-&cF;4#DtiLO#qs)J6FJ;o z5niBviiiyQ?Fcw9L?(gMnh9AQ4#$B}$`6&)j4a2Zns_aNe$mDmpD0rB}Y%u+ED-xy#DrJw<}6;fWF-G_)?ijFpDWB zGfKtRqU4<-I@K<>i@Jm?@+Q1hc-4xp8+os~*>cns7aD|vIZiN7p}cw3%0vyAhwVq? z3&v5&0+*tI++23TzDp`;EKHzSnH0hNMZ){9|LgzZEa(@diHiy|3*46_a< z)H)Pd1Met_D2)E(&CwfKm9fi`jOrEnfA*9;RsehowNRp4V)_lTK^s<0%_hoq-h!wx4C=vvQMN_v3znAl5MDssY!3<)_(YbvCDs!65h5vLbDyo`- z#xsj3WFQa$D}bUOp1wg)@)fjVPcDr}I{V2d{B&oq)gSD?LWNh}=acpekxT@!M-$)u z_V?cNcZYAJvmP`Dn2>jZf*%*HlorZvG=Zf?%fKj3#+O~BfRi`klA@??gdlMoX_8xp9l91#)2Ap7JOYy2Z332Xdq!%U$5vqp)B6trG5Xj*h zSVvStF%0aOR!b(1#yYuqASz5y>;~@jB*_;k31QGV0J=fPJ(tc0S=I^Oo5Tg;@NZQR_8Xj4lul&zSlDWU!(@WvP#ICca7I+_ z{jIM3pqr0hqu!+5AI7=N4mzZi4kO~Y$BCMtSChUWywEQ@&dxBENa?L8Ib3oWlg zyrEB(-*Jg)v;FP6@}1{NnV5iI-HRCjq?(SfFenaf>DV*KsDV_ibgKOEJ9ksOrJSwkZN?~Yku{U z|N8Jhj^Cg1aaiGzh9!|NbU(YpWNl;7G&vhDC>k?Pmms47J3&pJDyQ$^)j`)?0ltZ! zv3%f)V7y{HRLVpq$Q>>#t zvNvcd;A|A6bousjm6^ym&f7Ha|4Uyi$`u4U__P`jN5Gi4B zv|};qmTbsY_`WVs~kJkHIbNO-0HG z4g?v}&c2b!m0tF%UJC6+zE($<(MV`EvQgi73BoKuaKD>J+f^_6?ic8W2MJ>4e2cZ# z;ZXjt{c(sfry~1CM?CF+9E#eqT{Kg-lt+J*Cc$Kk6;wsyD$U-rLEp;W24-(s6hW8$ z$S9DR2^B@sLs6Bo2+7%`t)%?Xf{;a>`xe*k*wWICZBe>HOm~V;Icu*d%-uTAyYyp= zGVS$@mBDmeG#q^iMXe2M5sS=SPWHKyVdBZLD-MkkWI0+VDmO?Ymaj-AD824iz7{)# z6==LxyPTm{Gf9PtJ5lLy$c9ca=8tqWS7@o5B!yaAholx+d_~{ON}RFK$n9C<)^YA` z&toT1TZCS;RVIXG)7tcE`RB$Z zocLAW{fg8a*9+Pi)vui3F1GDLPGp$4&7>WboH@EnRjIlF@RXpzrz*tY|TFOuAg;S7-{8%#<|-=`Pa!U1yzXEM!sLq$YgSwGSkf0wTlJ8M^o zqFu7D$2M>)Q#!p^)~21&dM+i-q@qnlZ!&{}fLbv`mH8r??>h>J?W_7>Obpn`$NI{a zn9rAcMwk^vztkYzwZABTfHu_%bV^=a0B$%93B#Dm{ZFIOHso|esH!wgIs1#E+P?Sr zF@J1E5*X}a&y{1Jd;E7K-l2IzQDteQR0{Rw)2h3KeeuhIJ1da{%ztl|I)F#NxO3&x z;U_p9ecdP=esBv>7WqoxVB)dp5wfLW@(4~u}`tE`tD5?U0kJ@=`4QKVqVWReTPSn;@@=Dop zmHgPSi`w=-eykS0Txy0u>#yYsZD9J|auC_%}M!Um$6&Na>YgZ#RB{`#NG@P<-SIcoLR zt*p`UawHC?7DFql*$;N@BK9j_RY|>~k7^X&PKIBD&TWt;3ws6n)9=FAPZHX_o9o9NhAD}hOn}d*6Wb1zM9!^KK@LX9}74Q*t<*^+u zGhT_HqmM4-=y2h)u5xDqraIUJ19h_8JENsN>q)tkE-@-Wl7&+xKK2Zy+Y$|x%xsWb zTS+y=4UQ$y2p+C$)`&A@qOLP@EBcH+`%y+ox?4q8zxM{L^j3W`4?pMfJGwWb0J+a~ z^u0+&RaJb%jgabH=P0P6Mp5w@-6hgG>eqJUc+_AX=5%48<^z*Q@QCy$jfSzlK}7$v z!Z=c|>>t!h2@Q-Uw>6;L+6HkShLyF9EDgoF+ZbQII1>76Az^zp5>|8awP~%95vTmY z8U`Jw^~P=7S=-QaSEEl;HmF#1y8%hu!jiNmm0P{^Z_$hTJT{OeEK36O4j9cz5JedM zie0T?%f9kfW#lq}ngr+G1=WTkqov!LU|hCjH3Ibqc>vXEi$3e{t@>qc+D|Xa|@zA6JO$Jdrg- zc3nL}9nK)h6&v@CrVpoXM_8*FZUYU$vHP^5^40rlg+#m_zi4DWT*s7nhhhB24c6lZ zx8BirVI#kBgY~&V1L4A)5geWCOsR$})v98myC!lJ(MLSN*hv#ju>(upm1xq2C!s!B zB^FepTMzK*N`MU(U`;8{vaGe55A#ko>f6c2&EaXb12Mw>9Y)xE;csVn{~}t#DidZy zI=TftnZ5S!(yL|$U9a!5(G+%(#v?Y_Ew{f0O|9pLxECYrUF584i9|Zp@piNDkSiy; ztV`i_Rg^8?4^`J~*;&6S%%_~ZaYCt!t|QFPlyUIkj58t!b7_W^z%=(iU45yD09cMM zu}b7PUS>J}{0s3s`h+#*SBPEPDdOA+$y7Otj^c0RU>)FM2DWOejcYa|%!PCWl zVs{qXA1RDFo{L!VA+3e=o&;@?7sJ_rW7caR$Rff+b=;KS19H>6ssZ;<2V#7&oL~BJ zIIn6zSD^h*!@l$RW$8JbDHMo0q~ze`m(wWCO6Y79edN$|3e83jGdJ?BXvRO3g$%8M zgWTa+Ir1QkOAXJtAw#3UdAnIu8?z$Q_&h&3)bnt>qQnwK%s0KmMlwN)OeGMAa^SG) z-{8$JO3Ff^N+Q@d;;I+jOBL*m7mcli6I|zsiXReycF&cU4>J&pG{S*ORrXEBH>!{aIZ+J$jJ^k_XOVoly~ki z`|b|sdBZSqoT0oxE*55Azh6wtjwmQqr1n~8XR>HFm-0S4Ym2lb@@|hn63?U}kA-$K zNTd;v*XqO#am;BTvmjM~bDe0}5@Kz>mwDZNDsXUaNHhxYuz469|ne&+PGUI3B zV8@ivU70OfOE8^@Ozs0N-?5KUC>-CI4@K2i!aem(ar-;MompJy`191E3+W- zM;6vakJ}xUYgB(*>a(;I@I9_YtMtPDvb;26z~vW4u+Ai!b+_s2mna8^?Aj>!tr!F6 z_PEA2ID(${oU?GEP@{unB~PhA`~5$_8Me3kus_&uKN7d)u_TT>Ffnll4mC{UFu3AM zmvqbCz8j2EM?!oOM30yy3Nh#i`oWwtd=pk&v^0`0P~wa*GwRLLhJqf;+gnm*`HchrN3W0G9KAi&mH>Ms(JR9uBWwV-Fy&P<%6jL!E-nAe%jNtgBg9+pmyrCz?Q=~8|qrAiq70OF*%&2mUg@~l3t>RF5 zcKr7A<@fK8-=FwzULSVsM1f)~+@E5bkzi!yOS3kAnme-#-#&3Kglni}V zzbl=Rt46)CxVXt8KwTH}TF498Y4~Xwr7~wf?bugaKZ%9ga14xC_pi5lVCi(yQ%~Sw zYCmf^&oRfuOTZ*y!FhwZ0owOC{r=Wqiz|!JwbD$VxIK}OX zK&W4c?&G&d#Ow8ZT^7_`9_W_GTp88qtUE!TmQnK7oJwARUeM#=$~wwp6zz+hpo47f ziiLL=aCGp`yW_X$p889-H-A?Nifs!F{Bbj){WXioJCL~ zKj!hPqqkbA;to!3ACbhS0az`iHJO;T#|to> z2?q{FE(ppL?d7=D-byRQdqF*v{iJFjGhAI#dvG%vK%%%Gg_ zgIuKca9SxunFwX9;1YfSTtK70B6nk3RsLaEWW;zf^mgo|;8d4nkMjGWI!^6U%ppL< z9=+ECRc1&B_X^DkX$W6Fg|!OKvG@%MTO@qY^7R;2zdMk2rdg8Z@lAJ(lK3kf8VMbk zH?kwZa#=^3rIun)6G%Z0eae`lPIX|{$gWkfr_I5)zeh6;-^=fKn>#)2!GML?8_@ho z7&7@5=2b}XFx4&micCy6yCeVT?cvkcN6*Uz32kktbm&)n zQa<>G-fl%(*P2)FCYq6AE@hta&^-Vr6q$QsSO)#Dic3B4jtk4z>TEbz^CQ;s-*hOo zuA$DP(X0@8MXt-DA{b}Qp)8qGP;bSK<6VZE%FLJ~wp*6yDK@!oprTJ#)=rR%!_QPc z<=l{_^^~Wc*lgn){enZ0V6t5Py_|irWN{gq6joVyz$qlT>JKQ{tW!-xDp`<_=(V!c zT;Es*Dq|#D>c}LC2MZ8w0W{=-TUke{lg8B)Q=O8xRM2n+>W+4j7x21IIC&-~ zHYP_n(?O8;N;8`m^sFH0UA~-mMJj9C<=HCRM1l2^sh=uy<;e^gPHbG7DQt_dPhJDZ zLzZM*%O+bRYKaV0w>lM z4y2Pepmyl#dp1#2^T{d5!du(C)MM3>oel#SD9!P%8%4Nj_>4hI3y!^U!cIS1Ko z!!BnmE8f8v&X++t;Uw8e6vMvv;`QMv$p?@pMP?CczWdjeEbd+I`;0+&hID}uwKs*m=5gxdEq_N zm>6#;3bespMLdGeI)(}}Eq92{!cZ`C9blwRHl7(~sUIWvwP~DSqk1&4%0HF5lBO4% z|5Hrhz;IGweKkk+XR1%*NI~5&ZFG$xrQ>K}X@9OJCB@bWX5su+2^8*^b7N2{I!wz_ zSh)01kf?L%eUkp4a{fIE{NP-_Rj&=pdvznW<5s;#0fFu6n7ux%Y6e>^mPn@njhg;2 zR;T(k=nGXZDN!s4RpaX6QY;I_5>>SzCe@(HijH_Q&?c(H2`Sb1b`tcSz2~1*EncaEoHcu=Pp$rLfl3hwYD2bGGZU^|7A3EBI^p!8Gl0Q#)TbD@}S366;QQ zI3R~k#}^S^qF<)u29iZTK;BZNA2myn6gX%AT4j*_w>x`M7Tr2|gM~e;g5EYLS)h~- zCoFg*i%3urQ(5MdM&YRfq38snLhBaxZPB+B6~{YZXlw)rlhkDy`ucH>*a(*MCI0&{N4blCLx-2 zi|-zmKlmi>*2y--8{kG%N)B^yhzL^M{-~!23etrD1Fd0L_09FmKgPr@MMT-w%vLpi zNta8Gji(>B*?}gEsj6HV6)D@4A(AbZpDPkxFb=kq-W#C2S6zyi*uqdTQWx^zhV%nx zg*08{w@kg-3&n!Vy&Uz>C5T=}r(cY_XH6gd7@&`?N0k;XclPb-? zO!<=e*Z8?e5iS@v6>S!U6&>X@X2E4B?53@jOG7}L3&kf4Tm6n64Aj<1l5Uh>j5uB? z!4l$PuvjJpuPGw7#JHr)fOSgoeM<)_%1$(c#EokKVfgJ@(CCXgkUMLE+i`=+{7V^a zBi7uUQu8${`{P?$s9h;muPlZM$$}QR+`7KDjt$FA9T}hg+ALWbScfvTIW-|j;Fv~< z53O(W#oaLN)R)~Vavj<$oW=d7=(Wn3R8DsAFEQJ(EcKwx(TOj*)11SoY z3V@b^O6^@kacJNMYJsD!Qtw-d( z1^LILB#kH?g&Busv(Gov(<QyANE&#*mr*z*AG?}{xUz-$?1>c9!EwRK1GHKEk%}>Lb7l)8<3k4}B3EW6%V9w&p$6AG6cd7|+cGWoYMFAu-fVCL|@U!A-(TaQ+n+h0jBMHscaGOGb6)J$h z3Ut=aE9G2x+OR{pbOq&_{i#(&1(s1~Q;z3hhYxo&B-@`P>owkyj=I5<%i>#Ezr zc9-Ia{XqMs<8i7}{dnM;H<$!7ns^w3Nw!yVwzR#gi0`I*bb$6e^WM>c9ux|n>R>-e z7biPh+I(+07-jqz`VkE%`l;wck0;VwuXPC2#T%DubO=$QuEQk!zp2JNy^{vE~KCBYbw%83<95y3a zBTC93aykg-jkp*{E+0550uX~HG7SV@sF6Mt^cCrgO(P?$Lkk?CKHyFDS+__}qLs7T8@h-#l$~eYY|QS@zRj5QAEVYi#{dF7auS9`DPzoCzM(Nlth3P z<>V#@*?l4JrN=-0rDaFTc;YB3sJf%}O1li0aN&aMs28xZ zCoPF`(P^dSS?*FXy(hJam3vm`bBCV{YP=rp%CHBY3Y!>Bw z*t9Gm>x4XgpMdcW-*T z)U|(CyAdwA*^w4vtpLOOsxOTI9XB{g1QVLTQJI4O0(Lgo}t&o5A^# z+0ZcV=PH_D4goglr~<$g;zDZ~%tG%^$M1>^Qs=NtOS}gcl;Gh91^VcG4bw-SUbt7J zhM1FCR%TU1`z)qvC;{Z$MWB;)i6Tf`pzDq(OyKFz-{UvjE5}u!R1@q#OuXS3 zq-4Wz-JAWS(t*_3Wa@QuHrJ`H9-E*YRB|C=qe#CNUsc&=T$KTm*#5d(zGlC7>0i zga&4aMwM|ib4hKTp>?b%#x!dz59``Ptkvj93h1n97pya5)EKKa6&W_dEN?E{DNa!x{Q^VNxO*S70q{vl_verfn3gA2!f7GSYQJ~!&I7Js9~yZ>T^~D znX*XU+Ut2Ii2PBKT;gSW)HtKk*p~AMj$Cx1o=2ER7>-jI<1EB`$SA~3BFbt!&t(F? zYj{uhtA0r${qC5&?V8e+>T|q?r}X7y9&^sV6muT^`3g;Ql-L-PC-z*)UCV`s4cc4; zb56NxnnxbiS%=hzFa}VJFXg9b4O#gi8oKmvpGdQoOV%?x5e>@&!FPJot~M`c@6|Ro z4Wt`m7>K)z{t^txm3wfvZ71Hv7usTGGc)wk+5 zg4JIQOG8~vtLmiKD=*bG*4MR&tCcPKwJpk787#bD7}LrYb$wjaeO^xMZOs3o?rk~U zxB0eiP;t7RRJ9(t=Z(wuGz(Kxg(l(`BwEHp;h?K{<<#dd8aCt#$a|4PO$#<<`xa?!fp zapM?*_^zu!J!}eKF{Ui6hJ954sVsR#-^kO|ve=5Ng#C|oHAQsHTHi|r0Q+!J7zK~I zow2G(e`OOk39m2*v#Ao)IUeiYF1lE(xCNKhQ$-b2BcMg=w#fp9T5T-E+}!R^sB$c> zJgksYR`4u18yI80xCz4n;!^0Zb?KWLe=YyNu4BSdNvvtp#Lli3td6Avu2vnmE3U`a zAQb3|mh`b&iwe3z!`*mXSHX8sw}E)m3b71{hDkt5YU6PoZ%!@+OB7sV6T#J143nSK zaxrBOGlCboJH6fQ!Tw<9+i+)9b)0jyZPjqxTkT<6v@NT-uo76XxM>-roJ&gN+xJ>F zOG53hZcZK&Y3f=tQmdg;${o7pVt{ptaGS(l304R+*lsYEcB2x<8UYeoA8Mg*AYM#S zm{B-vQnstcJYwX~ZMfEg(if3anI_YzM9RI=nCMrt2yDvDy{cwkAy%5Yq2vv^x+=;- zHvlQ5$z~wWo#skY7&IYiI&YQE%@Sd?LflL;wN&OOE2Q=2n(k^XceRGQTDx7X*{;@V zS8KE@wb{IS26b9&y#`ySz1C~4^;&DC##*hdR%@!&T57e1>eNo}R04aIM!J?ZdRI;K zZd&MC8fbG2HPAdA?6`@QmZQHW-YaFNKH!k3Z8I9|Iw*xm*?@#G{E;5f#outCAH{wt zNA)Y=>E}1yLCNNg$-(KKzO24PNwG1Y48V2!psRIcL|L0to5CO|R4YkK#(nC9$~sUL zq^`&VK*a)EMrPw7rJm#uzRAj2VB95^Fv~!Y>obSB7NwR-;}N))ZUD04C&h17L&!_g z30~f!iUNc-7uJiQH<_ElyMQOCk}!%G$Ng_IRcr!N7{rd;G^im&x!2m|O!=ZYqQ2d8 zWFk7RwnArP=~r^UA1_nikSOgiA}gXR4X^x3QVEXUn`UNb6TDSfYdxn{>ke!Y4{9e` z_nzauEPIH{qOTf0R!$KI6I7WHBs~10kzt0gM~jvfE)fui6hx=ysekpP+)V|WHv4KY zxVb+MuWQX-lSJCHs@sTYz*Xk6`v?y&cUcORwR&yIrkZkzV&68{7ZB%}is;Kynv+gBxky9NOtM}vxOxuv1vbU}bH0LEm-dBD z1O2wf18IwhfM0#*)U02*`>G8>#j52&Cy%_%6(7Nv_3spadxaT3zdt&u9h%-@J!HrMYe$ido2*QM$KN%whutRBd0kWN8dfGIniY+jY+UyOHn1zL6K3uB zZv9Vo8h^9OELOVIut`)?mNekeZhQ3=&AQm|Mm}eqNbVXdZWGd7W5(5=f(kgb7+wUTt3SzUR>q15enq*8hdVJIHr-S zubH=}Wd~PU5`8nfx#85SsNB@Ri_p|PfFl)w-VFyR9D8ALQ}V83HQme2+BXD?0eW(V zk2QP2LGw{jLbR1`{>oq@KUyhN)e$cl5!UiMmTd?)hR3zxxod)Tp=Est3d-$1-7KZE z`yiLPgH!&vKG^p9pCE2GS1A;Cbfx|GT+P8h)pAyQE#)je-Wt8PKr4Bn#g&EWt=7{v zi53%C8UAw*>#;88oz7hy^J`+jG#eM7?rjEjuH@pFTA|9WFprCKL>jJE3qjJ}s@cZY zmUM4p8Df)ZR?*oCPWhmeXJi&+SaFNe=X0t(P9s1frns1+%1*LyN=cq;WJ$VY>QCbU zFQ<@*htTiQdQqeV*(_}Dot5EFa+y}??M4#eyDHpHSLgbgtTHI5^y1a+M3ryV`f+2Y zgX-yOEg234N3Ayv)eTNpqp$U1q&_$a0XdKy&Br@OE{QD59}19YQoL$l`XOwr7jC`2 z?oIoRccdiSMw@Ld>)36KaDUH@(d07UYL2`j`gCP-52x*-!U0pWFY=L3rX7l7xDU9A1n|l9e)7fZYT23jrRvx&In0b2? zGB&626}J_9rd`apbYS@Z$jQQc zh3}BB0;~toP<3)yb*Y8f2+&P~6`-jmIYPxt2Fuyp5w2MVjUQ@$tD@Gc2*;$<1zcpQ zu<)-l-tH3&b>h^5Dg{=6h5^8OYdOamW`YJ^1=8U;4jNhwQ={LEf*-p6CQF`kSe-ps(!7RPDrRI*T z{|s}hAjTq|NP%Yu6XS5n~dGVvO!Bkqhb;~d4<-6BPpUsCiXE=A9KvyAhoyK>-EZ{a#w#+Yb7 zoEQPUGTlC}30+yll!d4{qvO1{QZ8%~5c{M)9`h}uURea=D8Ioyn`KLx{1P|~{1Ygm2Ie$UltIR?V48CbyoVk2 zsxB853+_sNsB`d&42Wx8brZE$1f$~!AJWq%A}OkOjWZS2E{>7m3bi^%Kls?I^gB0? zFQQ#s6Zk}(y(R+UHPtANNm&1C?C3F07**R zTo5c7NrCcuj7??hq_i4zk)o3DfF%Cb@}C}_KKsExd42fo)t0`*Tqh;MR7_lEh3&Ca z+IuwtwVL7iPPE0A#{cwMlFxxwUYUTT{pIi0Uf$+v-sP?CRN3Mf#lD?|(HZT#HNR)wlecXX$DprQe%`?&A0$nWurM6r}D} z7CC=NQVlYt1**tmjUu+W2bUy55O9dnNAZjaV7Uyw;Vybxie*tnUp1r*C=V?sg81brAiLZ!W7+~bN@0-=V4r~7e+pvESjGK z;<3yX7yPZf$g<`EZx3wPduI+ZO4p?vv;3(G4S39K9jZjlvAw zoM{lpVf@GoW(ml}e-1OVlRr^u(ARiB1gUrGdACAup zTqDG5qquHyFrspOI5{<#N8T7z2jl`6QZNm#yRb-G&|jK&U)jlSA9znO125nl7Z@m) zUHf6$<(2lI9UuRQUWC&l|L5a(&rdLRJcpUb39Q*wR2rTTiKvyR-AM~g-~emEB}U8_ zh5e{!8K54+RciMKCZQC*4XwC5ow)hVjFcqEdPQr1F%!gfwR{?CE5gbc!dPe8-5c3~om=b}k zfj9GD7LA$2Mq!*>mo_I35Vy%Fn&7>Nx5+&0KFMzu=r~RT07V;7zzh_`vuJwOt2#Kh zOXOHXg{mBq;~f=ju6%n5jC1H=;aJB@%nnig7Ida*^tt+db`eeUn$M#vQ0PWcb-_nE z?fvs=_uC|5D@6syK-#I#?tUH3=g4=f-lyR#h~~g){LAVwU|#adl9dT)=XEpS2H~g5 z2Ini%al@fdpRPNpp5C|y^U1=8Pctk#Ufi)GizP8me9xzlYDKcicGe{TR~Yd;#B$JZ z5+@*_QNC2pkcl%!F3+VY!q34tk8i30364wVzKqw49B)Jn35J-$M#uY@a*0tybP@#0 zmJDEb`wx3v90Rdijf`WW6yf%JMRgpUI~$Z%*e{steG-Q}IdqY6Ichor+@81ma|6E# z8h(R`s)QBQ+JFtgc#)?qW~B7H;}v?Q(D+`wKRJ3X1n<-1)5F*8_MJ?4wPjjdIc8mM zCQMO7#y1%)V<)7kt15rB>n|%YWI^7<1(2rj{$AD8m4NaOc;Bv~o5zOI-ae=;!6d5O zS9|uFo11L4$EQP?5KDZs0`ocwI;(;Du;iCk#J>+hW=okaB4mMw} zE(D{Qw_Qw0p8OyB$OO2%<|m5tR9Wq0Ykuw=zd!x){b~Dd>GYdo3J-^P1X z??Pzm0%&OaZ(2up4aC-jij<8k78~5ITHTWI#rm*vBb3c9)|Z2-FKvU88hqP({8&C5 zR0S?s#;VURUcA`GKUJGi*?#0qtE6+?2BZ;`*In^ttz}5t5E{aUdUv%2mgcLASJ8cw zdDw7|-BRvrF8?)@1D*09NJKq)SmjBvZ!HUa_T>sT=@b&HZ^xu_N&*5ou{VU+sgMVTc{2SN&;#~8; z2S+S1#C2KVixM!2D@49`vvzsXd`R6&`n{CNZ$T6%1MrRQ z3#M#lcHoczXg0qJ)7;M!f60_tVQxUzEs}8U;fX+>h~2T`z~n4{pMn3fLwKC0qMWNL zgIJd)qJdUyjcYh;Hn1jx>o*(Rmkk$Phpz`u(@XlG%&mnSVkO#41Y)o(ni z;(#s2_vDF&rTZD!ukK{ABafO&Vs=DQQz7hM@Fe5lt8;aQ>7GuuN_#c*cPqC*waHpQ zT+=J?WsY9PrX()|o6T;Ntvs;rf%w2%(wUPDi5^>U0T4fM~x~elP^V1M8&S&Vlj_^B#Ry})tboloDkF^q$ zVmG<ZINlxr-$N+LT#OUVG3I>mA&~@hebZ-|DE-Gc;s=aNO=_X>QVXGYP0EBdR!! zR^4zyaSu@0t_&Wg7+P%3gcii{4TXJVwD0sQdoZfkdi1X>;9U_GD=y85uv)ws@klYW zE*Um(`_U~QjByq>u(Be~)XAjisvcbiqPR@H`vVjeGn1p7nVGzeMfZU&5@4W=3+8Mt z<>`Uf?@7%Gm;=aKd33%^mKonGqMRUHv|eLrFs%X|PM}C`Kp9)ojzf3DC{Po?s8C!u z(7XCgqE_iDk%H#Xro1il>EXL~FMm3G%~#6gCADN;0gy(5x6F)dM;RYVfrmU6mq|hq zsUDLBeshOtQN#*zClqX(E1dVFXx^Uf%3*i(jd3TQr{8(e*q&>(20bZP$b6042ih<` z@`glu{h;n4Z6f>wZ#Zx2Jaff;^LSps_hlKV_Dn_F$P znm|7>>{>$tGI|8lk*Ah*DP6wyUmu>F`p-@e{WnMNzCZHc9R5js zaOr3RaPeO9+P%`3 z&b3lQ`+;_xprx8>!U3% z_|Wmp_@cDPL&r0HG}clN?FU-h(t;05PqfzMg&&q5iS}Y|(;g1eb3cpDTMs#{3y=$R z0lfE=ep&>RFBfsxYVlurFS`g9A!lfTUlYuV(r&XwknDL3QSZ&<72E(=#!YJ;%wU6L zsNC{XYaEydiMsuejD7($bi7`#_fd>aBweBDA|Ui7{-r?8#-|MpQ5)CBuSIVB%wpr` zB^_7hqi{Yk9wd?Z^U8#8{WKeA3ie;ksZ6l=3h3h79m*1hvqF@XXNvY+*x)V3S`S2l zS=)Pp%!874$YWD3AY&X0w_;Z3X#i>^!jV|-7y}sUKlEkfqn%f?#T>sw=Qfg4)k(9KtvGG011Kli`O4~DLiSkDM!Dj&&vxU$AyX05g6tV-Q@ky6+IeZHE zVW{utaYPQR5oZ1&2|_e#wKvgx-{BM!S|5~V8|~KbpC5KfMCrmN1T0}U47G*dgr7m> zqig_7wI0tGP2KSg8h@j_-Sgfh*WFPVP}%@=ev8?AfFX%eu@K3%2k3;?Q8LdS3GBcl z>SS5uq@YZHnCOpF7|rvZLLmR+0l6E+OUjZ2eKS#T-7ZQdAsis1WOBo)&L>d{`=|Sg z6M;)mgbnFkwY?>132)iSSY$-Ato9`#+1bkQV`K@^=uVDa^i^cbQ8rGakr?mE8!W69 zDmWqp#s@|X{A-@KDIYuIX`J))vYz)6gh>R3KGbYE#Q@*ka8q*j-??|xdg>CDUy>hdd1RE-~;_;B>Q+60FpJ4^g)4t zEC7;OCmdc*?(e}~O1g61NMdY7Bj3u&8=zszQQy&_lHd(v#ioPo(NcK$& z1aTuo29oMUG7qzyyh4N<9V~f15VypI0`GBuuY2;wn}k=OanP{rr3{J*l6a`OLnNt| zk|b`J$HG;A+h-t`d%GK@q(cJ5Hhq*GgSBvdF>asfRqIA@%rLI*>%cWBySIG`j@ppW z34T>Gq7tZ*;tV zr$A9hjC06;88gsM0IdUYglTf8Hp-#?!_fwhykT+p6%9ldO?LxB28-ilEOKtaKEiZQ zEjTAwbnj?NkyGu#-Kd?eKPo3PGjH~xZgSQF^4m^DdB#fjxH?uX=7i-oFh3(~b6^$> zo(>KW&Sud(ngOqwEN8O_8S^o8kzPma0TO2)Bg6}qdD6|UBbX;ercm%n7~uzWTwwYr zmhg!&BX)2OPmfQ3I4GbM-gKTK94qBqloS(a9$qTSW=ou4TMUntk1e*-ON&NFSLvA{Jsk=yDHZ3c%C$ypJwEIj_jD$ojJnn)AJ55+wPy+Z~_nBf?Nd7kvV6Ph*f zd3jN?b%a*ky%3(3Y~$%dVAvv8J$f?YRIp z%OQO4-#)|+bVUy>b9BQRCHio-$ZuMh&Jjo#b%r57pRx(8BTknglC=dSS_y>Ky8;B0 ziG+Yy%UZ2LQwSCZ=j5POYe*6~uHz~!h#Z%9#e)OP+L#b?Dea-AfQ5>|^X!O$c4QJS z$?VKiOK>}-{h?@cG)}61v($qqF3;+38aKabq@*K9@iHrJL&^R~8#U^^laVlMcRly0>s zrfSNjT9c0pnBoZ{>+tP=@Mn>*?0)F01^Wp6DpAo%JYaxseXtIXsU+zq0Q^UWc$3CBfdI8LcfDJ|Qg z#$}{Ok}eWhfdgQ#Lfb@O(B3-_=iMB#gVS|kb6C6zrc z00ZU+#;n)Nja)l zXH`jPH5@xX;{L+>OK!M(JUm{RStZfkGXt~RP??bt9uXcM{`O;Wf8I^Xd%Ni)uAP?R z3mb#(TK9G^{RyNJGZZ8nsiWNPH1(+^@6-B5kp=w7qfT%5-Z*Ik_Y$sa7t+$9Z!GtUM*BUA*J*R)K!7+N(LvK8|*=TEl8H2Fw5Gn z-aE#cIx7t`LfU;MeeR; z6x7oK^?uN_L-NUJaeBD~t) zBkSZcGs@#vUr2KCdI_z$QQ5MMxXYLMuC<8sg{(Xef{nOs=NFK|y{oyhRNt$|^GacN z1y0n0;Q%fH^4roi+rW9M%Z1oY&A6n<=Bc}|{0Kawveht-Zz+qUJ8Yz&qvEjjVNeLI z05zsy(!dLHK7~r-%7Vk%EB11XVlYfQ=^hFm=M>24@~)sRE3t~nQ+q}D%{*Nz_em*J zA5tJlpJ5n9><}Lyb}^2M8Rle6P|OwJQNL2+#NzO9??8g)iyZE2P>Kj-I4lLeTn?yo zgjOOi(P67k{sW}bq|iZ0R-n=cy3{JUjcaaEjj>51 zZP;j~o$$2ZMq9BbfT&cMGUZ}M=k=QU9jGP1WX91d9;!Zv)KKXn9mk?+q(;Xh(NA?k zWqioUpj^Q6MvLhWu>nm0Sd4T(4 zFJ>SI?*cKc`Q*G937#omPgWzlg`;_g>A{RLqdJpO#$7J&&nO}e3CG0h$Y7W0!Si3NYTVP5F^pk)|)DHwxf+dVAxjk$L=P~wR7 zVvLtsE4%U&Z(-@dkqO%dqI&I=gh`5-ageBi1rfy7G7;64KrC_9eaUY>_Rg}q{g+}H zhWq?&boow^n&oGpTfO#QI!P#2ld;TcA}Ms zG~L3@U<+n8=lUzgG2B})Gf=~d`CxuZZeRtpYMVzp?dffNAJAS~aZJIcN=MB-BKOQp z-&-31CmsZ$Zk&;I9YbjufRi0{!z%uuCL3;?n7vMk4$h^L7#ev?2u5HeE2mJS1ggA{ zW=1h5ullG(Fg{QwoD;PkWaI=&P*hkhW2qfD1Ro1clxDFjJAQco{zGvla)`$-XG)Xs z;C>IZ52OwO{iq4;>^>a+`>rzbkdF!ECQVLuK~J5JQM`~WXDOx(OP>p@tHww3F_j7g zpEpojgC3h?M$o3<3O34#hdK9I%+EF}0z`IU=M{m4OJf}JP*5iyE6dlAMury8n zm<%6)o@~c7RjyV)+raIn7&bL*Y&fKbj>Bl$~`pMF!4Ze*)9;}|O$NM9xb#2xDeU1r! zCsMg=I{8-d)EjkUQOfHF(uh^UmbvT(1*n&woRmp=5UbF3zvJXNt?Y zfnKgT9T{5PzKf%E1x}bFxT=6C75TL5c&ssZG;y|BNKF^K!j+bWciII9ab*CqU!1`w z*0FQmFY!H{CHob)nY&gSy9IcjD}v;9OHSx60drcRUwVBTa8)Ojfb;xZK0kM$hUbJJ z&WlpLaC!oVc4Cy(bIGXNKI1j6ca9SscO_>7*}Qz5gQ(>qVw^;KZbJ3oXtV+?YIubR zzYtT$#V+><6A_x1ck8?0MDKZ?iqbaPN_vJ^6tdbX_HR1L|Iuu=XVNf)B2$;iO5eSH zn!HL|+Hk2CWATMU-NpoqoYZgKR=cU$@qCzk`}*m@U06X|(=n%vJ9>>@k@XJbTi^oU z$(8HTx};>R%!W6kq5q<&ZC2E7tri(_O~@Cqyq#FSwJiPY&G!c{|MKITANNP%kZmhe zyCqPDzpONg8ZBg&lDmpIk}M9|MS+Gqk|u1&0bL$-@#|6`PsrJ|hfYv!PBHbeoCJGm z*~b_0^4cJ&&AgtNp*Hh+z36M`dYy20vMy!IGSv+}|H`}nu`WB)z8 zckjXO*XkKPAKt(J^}qct+4&_FU=6fE4F4~>|JoOJxyY+!(O2>)*ji62a@ts~tK0TYi! z0a8otLCqbblkWck&~88c<~H=P05{HCpv+0ZbBerE=a9;tqNPB<8y)=Do6+kRKfQQ8 z+JF7>*^AM^n^!Ns-^Z|=dk^nh4ZeH&S84>W@9kRezJ2rh`RKbh&tHsQeE$=~jb_OU4QrTn-(9drZt^BfAQ_pA73Acj&{F((2N6Bfr%)f`htZ5%D;wV zqH1d8)*2V!wxQ6^ZL^*=bR@&rZQTiM!0y98{2rC@QmgOt53poYMT0X#m@duF+=xyv55FA8HwZg8jjkx;xN7y6xS$Be^B1( zr*k-(hsElmC}z?@h)yCNVLXc%9GH+OL**Z;>T)1&5_Eo=NChxyZ0!hI%B=*tid5;4 zS7`j8Z)7?#CuVdsv$KsUaz6$alHfZ5+;Tp{q))Kn2xF(+GT}unvz9$OUC++`Qq08# zQeEyD)E*rcC*@2A3hRfcO1B?T`fl@ZH5U?*_ccA`ZFMA}6^}ML0(eFz5+KLaK`+bk zM>okGSp-K&arrxQL7?r&`wX1-)I-$4Z?HBHuBYG;`05O0FB&-Fp}(d6_>TKQ&znf< zt};6Q$qxGr5=*d&&!B53kE3rh^G_$a91N4Gf4J`up3MByN@b@JfORfbNUylp*zGOQ zm?!)I$GFkzVm`uqbp+>jM0rxW%#ecF&{#-&m0w_duvo?U5?r&2*?Bj1jrEca1g>;a z)HW}=>0mOyC@OWN$K?|CX^b6WzO>Gc5XC#kpzno$%#V-Zui3mKGm~LB2yZ`x_bYKO zvH1-Cds!@1^YWO%n@M1}BR@@qp0kWQ`Xk78W&s)a7=Vm?%fStOISVJ|X>iRw?p(zd z{1Z7w9#X(iFWI?^Yk3zEgUsd_LDg|P8{UQWyDP-aOE^C#^Hp-CCh=AEsS^k#fY+09 zD)`K4d1zq*(}~3ylpc4mt-1|dh@*KhLzO`Ic7DsCTXYnVsK;QxoeXz& z@5{gV^(FWWhZuPVdXtPJo~PBii>`66)x^2#OD@hQ{V4qiZKh~Gn_Q|i)gY)>`D|Gh zm`y>+CX^bfM8Sf3!{EhAfCCv5szRn*K#%B^7Mt+Bc(s^=j6D^?z9ktQLNTC0TfSCqr}L$u8?C#Bpb3N!)xv6+IBBR01WC7lQ2dQ%Bzcbh0hpngZeCT!nOzfi)W; z%d%9w7+X!wwdhHOk0YC+VjWlE>dZMDQ^nFh*1U}jS1QtXL-$-iVT+ zL956*OWZAIBuR>f{chjdrn`5OpvH!+g#P-8H7Z*a2N*Gi`A7_)5j?fA-MCPpOe8<` zPKfBip(?2@Q46ZSas_ITrQf~xWoaV|1JyCIn4mcoBqr?|aCr0*K7n;^A$VHfK z@3d!a0sqL_nmd^Vib9ZEcp`WnCH!sWQG!s0Os=0IzhQL$TJ+J7({WaRX3QCMdbuXn87l@Udq%u{_``J#{abDye6sKti8wJY7!J2UDs* zI${ub_H*#1u$~BF!+u+waD-gMTMDH#r31Bpb$l;xg%35+TONi204i*gPg9!ou2j1 z>ICY$ql+Hz@v*m*CIboB3QzYj2~)e&bN)SWL*i-Fv3{ztbp{wy)>OiTy7K)Tkc z=2(jPdL?vL#NQH%Z0NcEDe0O?Jofdk?9(|MqO3o(H99C+7-lKZH30lX@G06Q^DTuV zWB8}coMX}&EHR(&a*@wS6c$3~EV-ZjrTk`)yebOObUYTUu8vxsLiXSX_xpz>CIkX0 zb-q54>2~KQ$o``b=*#Cw)u+p1axTSjo^FuTw3oUPj+Sl&_=0EXiUk+VAKV9~FO1R( zDV$C$#Iw(f_WM^?u)nJoJu;8)`0&2lqjY@lc`J@zx&bDF;BfFStS-?tKiPZlA`stn z-K~czJC5#eGJt&YMc0pU6%@}sEOBUC0D0zlKsk8t0&N5lO{rsBs8}FI(?7?~xsNfR$YZzIlTkI$t(+Gt%4J6&?Z6C|q*ECYhw%Kiu2Y^Z=rjH&0 zsRrn&g;crlA!je#_%d6r9-tq(Ogzns^|^ou-PhMspj1NU!GD@2Jwbna-aS^km5MiU z=OkK7=XHLKf)hK@$a}k~Gx0FqXlBP9fJ1HdleV($O1mH00KbzyK}Nf)o}j=?PE+5$ zeVBarpHH>J6?rmfCGLk0f)^3;m0%h{n*N!*IX`>=rG`!}sB%Ip=PgKT-y_|v@&1yy zv=legWSqdJmMi7SMFw{XLLY{ZxrZ^Q9mXe!mgvWk{7@`}lmJ&XQ9{7GNcj+dYD(NsZmN_wU}Mv{ZKpinsT)w#8KL^+eE&I)2ksF~-; zRTnU(j6?p$1QK_uW!LP#f|X~!WD$ol9ayE8nfLElHe_*S7XW4Dpqd&=pKlBt;qbRo z7X|@;4OwsrvY%*3NRYovYJawG7|){Xyr|t``{ctZjc)!6BjX1?dYo-$zzTE`(Ng;@ z1C}W$))-h*FQ1nwXk|A{H=aOnxTvB%GHTq$=Fi$kL?Vbcj)QermkS)~Q&M?`W#q_r zk{?~-X&M(}?c_?iM3>h?x!_u7@y+}PMAXlTexbHu&;pbFV;J7AuDy&|Xu;Hh9hc`N zCCfa#^v%^Fh_bfRj2GLi*OM&gDRi`}KbJas$KFFZ(ao!I3L z>E(bju!D=5=#+G&EJYFyMgc@*m>TJ%G+plVqjCw3%6q%a`?$-`juYi=(+N`|QT|4q z4Wk4s<)Ox3O4%;Z09)4@CeaLPyb|CF>R-WUpA>>p*+-dC9&10MQPLg|V_7yP*{r9Yj&^F+omC$~4Z6RiAynCEHl*#mG<@ zZ#5|Ssp+M*n@|R3I@j1ypOj|>*iodGXj(3pkPV#OY-JOPaP#s?BHXr>|c=hZLdAefh-? zBY*|o{rF-Z9Qt-TDIDd_Q1+I3Mj(@_*xzuS3o~=BA^S_hihOq24fFvnS1I~k)Su=~ z=v#HNxqlUNqsTvw9FK_Ftjsb6tFa)roWT=sq`-ur38WAw7($B)rcm`^y1|_u0trYR zB(zm^bFSp@pGfLuXG!O?^{#ip16;Wi`$Q_5Wzchx{5Qrr1-86_OYXWDnmFih6&VB- z?j2_vlD-vnX}{!_GsuVp1xG-R&N|1F`Mm1%+$6-Y?XN>(X+)4{K*~lNpeRx?_?#>P z5AO4KlujUg@YNg!>)BFBn8p7T+PXp%jXsfTGHEQZw}X8;4B@rc&z}-+PVPRz$!&9V z9m*Lz$WZ%$qa}JF$=MOkx)zj<+pR+R{*ZDI!rg)0$OIDN(o5E}qe%{>V-rrl<$dEK zwR>xNDv)j~)riC9h4|CJwb;N<2FK;uE(Pb4b-WhbLJJ&9w(Fe_?;CSMzOpL=HEp!l z6Jr5a;^PUpcxnH*L~?;19g=)$1g5ldmEu zxQQ~vUMnf&i)CG|7Z7T>WKL~Y!NESoeE9W;Vd8{2BRtHYTC#C*)XgqwWinSLcPHE> zdW(3OTN4<3bvtb0%69P13Gs~T{U-bEl3aACyNZnr6r0iO zP8+tHdaQ#tj1XbAFE;m0?=OGm>aq!s>)lnY=C1mTLmZxa=(dOy`3N>8Vi9^}r^5?= z?C5)27jFo57Sa2L+I|WbTcEH5|6&Op=!gy{qC&{RZgMqZSr$6e8#?_Oif7%cfx16V zgDjRHq8rGoLpj}J=Qxd)J!&hrrIBguS@`8MiLTX)b1e}J534@#V#N?A=xQDsYhGF1 zchY9q0z=<`=B%{_Eu={+%q$XHXgDlz@!=oG;CNQ6QwzAA2RTiFW{vqu0#G*}4C`7WmOGI*U<6JX}GtPS>57>?y)TNkdNRpxmkqw z4F=JZef_YMTD`ECD_`rHt4fFrt*aE}dY0~t^qY{D7|bW_^B{)?=q2+*ux0c@y}UQZ zI|+1v3=^ImTrLdpwu{t@I#_%x`$e{IzlgqqO1R`8|Z>ptSZd zhIY);=h5CZ9aF!nP_sa9kpmn6&21<;LlsAwdMd`?!3&^SxLpX9c}p9gRj|B1pbmET z!lUr`d8u>@2dFsI$nq-=s?yr1Ta4R{3SP*|*e7=)GCmNK(Y_?6=-|K!c(r2xZ zc3;@V3dGhc$I$?j^>3oI#qF~6{Cau`#T*$jN`(ll@hL#x6MeyLLaC}zSjeP)+6G?g zrlrm2DloIKGz>C{)0464i?tYjwiTE`{pZ;XSa4b4EvVa)m8-&&CQ#FmPF~X+inEQU z)#^IA#;mZ$!18G8tymp0swZInlB;EnheA|0#mr@%?WhtDIIH@c*u9q&HGq9OE1dV@S8mnA?c%PB!r~r z#Uofx?3`R)))1}XkHoX-_jpOUZ~WzFsw5uwU>?_5tl*DLcAW8obZ#fL5|!7b5p?Ph zUgXtcnca7TVU8CMN76G)pEJDtAaNJuTYoqz?oaO>&%6_5zHD`-oQr}L6j38UKgxpu zO+cZ`Cn-Ja57XAz7c}J)!;1v468xW+4@m$Q zWVymMh}wvrEp4MpG@VT5px&*Zx~*jFc!DJ36fQO&!PT79y?UQWY-~G$-n0BasW;#K zlRba8i*_0N#B6fg5)e9MP+zomkLseeN8ZkgAj^nLoBReg2g!5!SW8USC6iT8nq6Vx8CQX!KE-@qY(T>}FNu+MHNIz~t$NLm>~I@n zm+VLq3o$jw)>P}tGqfm*1xWfSUT2&dj<6jdmOX==_dRD$!}nZ?>n;a(RdRX`!L?nw zNVPT=K}BIUKw9Gq;clvgRO}e1$G8MGW~5sKi+e}ITr~~ysopiKabD^Ok67(CeXOH~ zJEJf1Z$k^y?e3o>_H4F6c}-KvqHhiGNv5GmewJ_T;Uf5rC~5=xRThV|aWCs3#2u#V zp&3e>YYR}R2EvL|nxr2KYZ&7tF=Ui4N4w*0kc%Q66ML}NiqJ`@H3~{6*Z$C+m3$vC zFEG)Hi4|uSQzke=#|6&k!q~S#Cj5euxfZkYvJ%3e;P1P2InwPb6Fv`{Z27QR(v&v6 z)@o>{e+>^H6Tv!`)fQ=BH#mMOw`$Gmu;i*UlTcOVPqnkA#uC3`r6^>moOR<#;u#H6 zt^eq{r*9gB2k=%infYYQbaG~lEB#M1Q!5ycxu(@Qo7tP^D=(m^q#nwV-gvX*4g;>A z3{!~GGan|m%@1?LNN9%qWIz{gM8;}XueUM)I!>Qt5y;r1jr?78vfN2&1UCsY=PQW*eah{a!xH!0+HP&slzv7!_q zZ%^}O(a%q2^W~~MBH0adBdAAvqR$Ev1wn}&eE*l%--$zVl&gdaph4hXs`ya4q*m$* zE1@t6Bq8dNgU&)s8%>CmlcBi4N+?u9LmiG2P&LYeD5CG`q`*d`AL%cu;<%hlR6D5< ze3D^m=JPUdE>M#Wk{>81u@8e-CaBvwlUXUSc$%^W2`son3$3Qvoy`UINm@fYaX6VD zlAkFiJGlT#MF<2KF;66XSV^W&U~&2+(;*UQQX}-iEC$5klz7(>)lmMtiigwn+ z7J;QU0nnA`+hsPnUeWn313kJm?Hj>S|!emu!niB5M* zCI$*{8JCqxB~7x@@}9u}8#h4&DV zi8@=~kK349wT(d8T)C;0G8)&qcyQuuyLs?&GRIdX{g-klZ_;V%yEeoB_FS2GnW=iR z%`3aD7mzQ%WxVciYTf&Q7eR5sRYmfk*soMjthZs`&sQ&}LP!PHq!{}qB}U$efM|;- zxEKcfRR(cqZ4OthD~s!otysJ{C6rm_Q|+TI7-{U*35SnD;{?bf&4v%U{`L+$pA9>t zI39`+8$As~5qp!5eH@L`H})f4XPb+%Je+Y6lD+);>=Dmf{GNtjAGpdUEzO(Ts(qi- zKHTN9=3C$Fidpv|aaij4@Zxs?*Y+*^a+?kbReooy)eEViu|G3?ES8cBMr%DYB@O64 z2gp@Oa@?brEezIZ<;m|P zC5PqU?c&Ka!Y9|8p>h%9-Cm7j%O1GOQC(PJKeToVV5vZqf&D9B)z3QFjcNcDzD83`$! zR1wS|KMx@RVF$-xbK~TwGTwfRx!y<|le7|`5CW^{W>QE8UT{{G(U~ZdNtusH%B9Ys zJHMDIM!(S161;vNDQ{p54S`x2kQAjvoRVyLQ+Y|0AV<0su2o`E398h_^6+sT7nI`< zkI3?r!Ag}$o0+Ly!R|XYJ-{u9vnk=D(Qv1Aa3Ci@mVi0JD!XuapoYA-nV7RfLe8ww zvrf{S?O$cBs&X{Nn<``16|yzp712{AfwC4QW^Gp#eyqnId&vlLm4^4si=3h~^t4UP z(*j)~-Jq$stl)WfWHcwY1x+!dzk2ntYB9>-R;VBkCeyq*Fh`er&n%Sf`FvOAlrwUa za~WrO{8m&yB4aF%8##A$rWzsCV08;LQKM0tq1$F!R3I4&QdP(%yERR~lewIgE6V*p zCQokvp!jsu#^-pCni@K%#SJ!G<>YpT*~KvoPn{kb$!?GaXQWH&1ip7#r2Ym46gk1C z;Tk5&Qa1j`LLBSuKei%1;688Od(K_k^lUcI+t{Yzdot9ON0yfNJk!jfmA@KUFZS?t3b;A&2xlBBq)862U0PW zz`f0rG585AXIKSE)PRk8jqqFTRZy6~GS>%WTM0R1!uG}Y|l#}#X zcE!Q3H+3SlL!bHpwc*`3OervdJ7O9k2~v`(b0!k&MW*24ArSIxkv)t z?5)olcNreCF1B$4ZbPmZ7UJ?qAtn^>AU~08jwWhJjHqK8l>rD)Tb0)uFTJx8c=um=#o=iS=#Ws|U!RZ-oQJrL{69gTT| zv+2YXDnLg)_*ml_*L}trr`sDy6h_I-#blm%^f&nJFa!H zn{BMTuO*8FqeLWA{};0r*nm*ZRxgX4v;dG>DFM@zEY#n{kp7RMU1;OO>#*@A2UUdD z{7D%pXWw?Va2Bj>bFCawndZ-HgM7iktyI_L}G+-q0p?e6xH@02Q2 z7I^%d^g}@N^LjQznhJxeewBe%+@$i0{tyi_NZ;tg8B`rT%_ql?WR+mB^;2usO0YbY z>h(a^{yeJY%O%xaM6>=Ll{dr(e_qP!cz7-mtUr&*@rZJAtL_%dVm+RtkHWKqr&zo- z2OWX*Cs3*sQ^-%P2x|YeVoP;Lbvy+pEZEaG2MD~Xt*Y4L^bw{(nUu34uPEu5c1oBg z`SHrkm0UE$&QWnmEbT1rH92Rvz_gQtq((xly@VuYq-;}{pz80aKu_mE@>WWTeYC2} z0=8#Gv7q&qS@;PPj!tu6DsX!QoT@JwfC&x)OI-^~qHU_wcUa;V0{ktCTvO89NsQys z?mG2GeLA1dUqkfTa=CUD4*M^;{JmEhQkTco<-_qn^BYRn$xZFDIeIU2WmCO(165%( zP#)Ga;xE|dYo3HLE4(17_rjDxGB1EPbO2<8&>n}VbTM75+!C{0U4qoAg-JxMXJxmQ zmt0RJWAX14XXl(b-3Sme3*AN_up&}4k@aK}h5#!n`IT6=2H}=O|3UHwwD}<8gOd|g zb}uRn>@y|zfB+g6ov0ft8xI&CLLKumgg47YOu#0uNNG|%i&YY^qU&0 zit~>O^Ik<-8yXW8=P409Fk7|dOf{q@e9RO4M|*DB-0FPz-YIg{VuW_zSi|hXCxE%F z2MK(!ok1on574CM@T-tFza4}D-xkN3>}q{T4^OGC{#QYRL~H@-f&(eabR?QSD61-7QsN{-lo zYP6G}q6g%cTCNrn*5_L-G?luD;>cOvTG<~(q_YhcD!455!kVA-%f6UGPO7$5I^@L~ zk6Y!JoSh%T$~Yv%ny$@E$}&S*=WaP97@fD(tkXlU{517kE{^Aw^ehK!O2o1%J>`k} ziF^{Uu4=v^s#CY^`jnyTGuZZGlMxoT)%j#p9Ajz6S;8Im$PDLE= zs2qO;eLdCVFsv&_A(Hg7#5sZV{Yfzwh*-HYhVDGm|N2Gc;KsDNRzu8>0!#u50p^jj z2HFtI+Mx#BZRb?Xq2^SWH`tyk(NfBIySW2~JpVRz0|{fJRKH4zZPNI%>BTRQTd4b? zv6X+8Nj3{!Q%i8P0`1VUT!O0(%drC&3dLl z823q(livG-?HTH*udd*p*?21YkRYT<1W&D?lWR!qT_2BmSWM;@5f4fqu>y63tTVpB zwZa^bbW*LJ&IzN_KebEaXOm{;XqCkhEX917f;Vs|6|kFzDf4gPUCeDdbTgHn zf3Rw!FDxu{)3AMRRQyd7A1=RS{KvKMc{yKKN3sN&MCw{1BMk%mT4|y)?W`OcY{L4V zvzRB)f7X9^7X9#5_}m#+O77ud#mZR=#yP3X`W5P5b2jK-X}L|Zcl13rN#kRBJy@d$ zzfQM4o04&q#f;iSNo>j}Nn)jgfT5lw52Xluy7(N6fWNGkE5!$+fMS9k;a z_r9v!DyQyg5aY3C)HJSW?3yvrSMraL$aYV0)ZCZE?y*i2xp|+)KiW48xXw5M!g#FK z(0_+&7jM=Ez-HLesKa+1tjyuyS^%px-4f!!9d;|(r?3<3a?fBWSeQ)-`S#t`~ zfYb_@8>HXu2Sn$)cdZfArY@`}j-O&Gt&6tbA`j9r2^hN9xq}t=$})mjED=%Zj1%wB z9T;&iFxgo*yHi`RqKqMsHXdfyj<;2o9d3UPwBD45#uRxnibt|Cp0XtR8KFSN@u7AC zn~m{5MK!0WZro^As?dT&(uvr60!yNHW}U1ks5B^M54UXhwI9II%-pkD&si9P8w!6G zX9fC=oD}6-Dh~_8AqtT)E|i|-Kg2EZGsFE0B2DYgO#s0e(X9VKW4TK{>pzo;s7a*Y zCuHljAjez@zwf2nt1aOaxZb`{z7S%VtF@W%#lzTsExPAlrN21!Ogb{Dp*ZUL}nL!Y!p0 zU=QJc7)iE^-2E$0dH0Q#Fv)$pZ|RLAZ=dChWqk{9n^Y={-Q+)COSJZS2hB`f)H%lldr+VNkzpMi|yKjuyI+olVhv zTU!#`G>h(zZlc0HQB!M|g>qVmA;oQCrCi&$K>sQBF@N3Qsv|FqfEm`iH1lYiODJ7QbZrn;XU|3 z7CFCrw5*Qq7L;WSk}BRsWI9?Wdwa>TxX$R@VqMkM1UmY0Su8QD@7sgFDwD~yn9i%q zWHJ|r>F#m`5oZEn=hO0NsjIMCVI^QPIW0xYlPaH*;RYi2raeeDB5o8tO5=wz*RtoNc|m}spk{Oy?CJv%*p=#)BbXG z2@HtVWapU7L>6sbFCoRJOcjWC*=TgU#`>}&b(>)-4*Y7VTOF@wN2~dKvQ&*G)tU&3 z7>S7=X%V=-x1HVx?HeCjlme~-q?iE4GwOT@1cMCpv?3%`9aP#&-H186WBPk_woMZzpE zqe*!;huONOnIu z&W~1fwL9aB<*CZq642I$fRK5VkH;*hoEo_4nYcS9`Nv|c%jk{69Vz?%N=UBr(b_y; z*=>Ee*Ww<~Aj|xXbY{uAfgH@ifBYbrg#FRm7e9=i9Xy3XQG&9I7c#^4=-|!USK{gJ z&i#kt>Hd#z-@f_bKrj*VdH3s~z$EylhiwEHqaU6=fB9GV=IaN4`2DTqM<^tL89DdA z8}9BvT$R|DS)qvlsc`klA0%jaJ)WOu^eUqBqxt>kchbXKU~qT2C&MadA@2ii+a7pnp&t zL*84Jepk@0qkIM>ZzqQWOSO4+^z6;kANF6+ltJhjcL3gyi;AbR>DK4BK z<}$;vNeQ(qra7c5UQ$>{p6CoGYLiRsQOM5RTb=+MB(EV?u@j;y1SbeiD`xZc$*I|M z9e{!TK^))tyc|Q4TMSY>%~uLeG!ZDI0wfu*qp<(oi|j_vg;Kma-07Pew^5vANE)E8Q@OHVz;GG-vPzTj)AV3`mP?P_)(@Xx>@KJ|1$xeI6{K09y94*C>7!?A$j{^4eayi26 z^`I0wI`dwOS&Zbpp5g4CE~n4w_dtk?PY(`QV+A|w9VTJtCy@I@3)rQlOQXAk@ehy+ zcTMeo%iA-ZpAQaW`%?WQ?+dbFe=ApVQ;=Xx#$7N+>?!`R9DkJm#vOQRZ2)#{aWwd` zxT(Ok4fCj@lE4D$+o*+e_d}HYZ~m|HyG^e;gGd<_lcw4JB4%t!5IX%*~Z*aL%k1| zTx;9C2J^j5z47~+gM{x#=Vi58=acy937CDt2TEI$?8nlp$7@bjs!1`kAD=Ta^6Q*P zZl2ok;@>etC$`T~nKT5^i%Z!9`8DWT_M!jS? z*CSmfhvFe+#*#v%Scr@Kk{8CI_FwX zBaUCr-9EHp3b>??8GhSPYz6wwgWv2^;O-MA1nKTG6V2HT{N(84T<_WJR|4<2FPnnl7f1a&Z<>oXzj@g&+NbtIX&#S&o%3kBmDb@d=RKa6ht&diqNOcia{1sU6M~ZH#^CdgO{6>PX zfYM#|Wefz%83aI|R;a@{FRH_QRZf%c40MPJANw>)O6o0|8Vl+yWJqN)Uw~F3pFl7h zWWkx_ltU_Csqs{gn4GN-k_7nwC#I^B1cbDJau|xS1#*v+9SfYGg~O$iCn096J%N;; zw_EZpDcKfdL`aP_S?k2NB{boc37!nTO!^H6pg*+v+Y@=ws=e3?M5!KOJveO16Zr;O z(sHQAee?P4+Ruod5sGJIfaywNl9Ko&9<=bGVImBfmp>f)ShJ8xXefMyjo(-voCAC(OQci z{eU%y??Av31W_riP#>1IW+6-z%mZkPvb;%uQ`PV^*GzFJ&oMxnrt$E2^0K)Ft@4B7Ud@ zt!f!joL>@8yq+QWTAG*WqjsoYOsEp6f@39m>6gl1h8T5l4HA1I<_LMqkO7y;S;p5@ z2f(7IHEJ?;I;*k$G~zUtu-L zhke2RB;V~nmotZcWF6`keef#TH}~I?@?F!1$K`~8kqZf1jknRIa;)t4li6Hhmz~WT zZ-v=h@6I6EhieKMOmsH}J&@&9l~+@JpZbg_6qKIN<||0^Dj?`#HS-J@;wMj*r{(b~ z{S(R>Vw?&2d0|&o(FdoTrJ^%fWt*lqr<}3R4xXl%aD&nnoHx+qOXh}7FfUb6xmPTN1Y>l4K;+iuYynA6H!%N7 z15E{>yeV4%*&1nX4mlW?m$RdhTnd%l4>*J3>o z^{XXsdlj2apRYc$w-=|7xVv5jO7yxUUvmSOYiK#FaI2XH8sMJ#SXeqc!kqH9T7+9N1UJ1-NT{o*-LCNHu3WfSl^1 z)RpM<N#voA zaF&zy(;NO`(;K#(-cEFSE$*-yCd6Sxh7F8})qG;{BK!{D?22u|Skd#*NznoYvEeFs z)0=_rXadl5N}TR2gj9V53@u#=?~}}46LHYGvdhQzn<%l5?TKJQ(wcAL8$-A%K-gl3 z;s_+VloC~l=NpEZw`LaDu#$hrm6Kyg+K1FV+M&ZJYj7qJRH&(gr7c*}_Uvd&Ry4wf*07-U>}Lb(*}!(zu$*=5W-Y5($7aS^%s6{l!&*kz z%4qo!#q($+^$+J_PKCX-GjkC2Jl>ExhmRjy8p1$IJ`~8QG2BvW=L2$76Cd>j@4w*v zCU`FeiGOCq*P&tpT}h29aiptU60ueY2I2@|Vm$?s38GfAq2_C#cli}0u`|KDx)1wH z>sRXaF_6w-BP>#q5dh}yqc&K~NN_{JG6}{9 z3V0yeQvy7oSKTy4!!@-N$S_QMPNEnaZ8!imX)LYf;uD|dl{gUjC1fpd`sjGG5+DjW3ivULYp3uUG|D^uDL6Q@kVSKyP?AYTGxuJ>wc6npR_k@L%tQTH-_|Pgy zcXoFqTohObgOgkzfd>tbL+F&js&N|JbHQ?tBWoa@snI8CWm@3n0+UdX_pLs7-^mUT^@mPyu7q znf#Ro8=YB=&fG>Ac#@f@Zs)bViK@<%{gi!k%a2eA)XH1WfODa0vsM5MdcK_IOKroX zK81`wE{=*N+_iRk38%ayd5%*Z%Go^mAoH|+*t5%;+>*BDB%fSBC0#0SOOfv?k1trK z>A5t+qIsF`y>`Emg#ZKwkdqGNYAL{BX(eD4fWu7)pJOTA)#)4z&C(x7*`LvTC_{;O zr}o;(sYB%TpOx!B9gpR}Mk(fKo* zL#rM4vBB#t3fXy;pmo1wE2dIAh$+ghD;xBZ6TvBlYs=ID{7FE3r9qAslwv+T<|a78 zOm*>_AR;d6lgoufKVPUPNBSuvBLZhE!(=QtsL30iUrLY;&P~)v^Yq%X%%D9d{8p7D z@oz%^0(9;@n65WA52x^6dijw9F&L4Ra3(spQWOVXI8r+01d3fu!NMRlDuy85gmEW` zKVckdP+ZhyE3Lz)4R9)oS2eg5$FDjZ+te8_fNzaBSA%!WxYvMxwK!OVhv|TlbUC;N z)yuLjTYAB&fqWiVn(Upc6`{uO-70p7@tKAA0Yb{lcK|K*jzSG34f!qF$0OE4o|D*X z#Z!jg$|Dea&5uxhn}&|OL({rpG^Eo$VE#LP#{9SJnE5ZQlNK0Ni8tXFXMqmok&8hY z4?uMfp8vKnhBb8jf-P#7njb>@t(AsC2=ycJe^Px#nnhA7LTDB@?;V!oKwDZuzo|)U zQI4(BGq-H<463*9C5vUT9?$y%Uf`AIQx&5*z?5i%fwE)B0_7(yh-u539zO+ZXGt z+s?X|Vo74K?sWtU98rHS!V&7O%_aCwZ5sE0pc_!PmKS2MDdw7E^L`R@E`TAaekKoh zrmh`cQk>HTWR;d6>-N6;oz?Pji1r6;VNUn{M`3Ub@Zw*g7!zs`MCUll)g0|mRw6@h zo{GC#kf9^Yt4Tx^uoHx0>8Nco4-h?FFTf)`n0)^hqPigFVy@&cPoK!jO?azAt-dve z^J7LyF(8yt6!%c5J0k64%JC>QCo0)|Rh&pac3FolTx?@px1e3dZV1X_9n{ohXbQT_ zncaF~2HLChOC>;b+bAAVpZc2`MY~SvxmA`2{ z5sQ!)-h`4{8|{=?@m6rhsEOpgT?WC|Rx^G)+}OR|{0wTAR{VBS^X<<(=a`|Fb)lk73{ao^49PPsGt;Ncz#4|K5jPV zn;z_oS5M&$=(IVzAJ_iY={)(UB#%=j8YZctA#5MDvHO%0*}*Ozc1T|jo`NgFy8vlq zZN4j}KY%OsJ2gR>6_F)te&3_K%)E>D{@}x3Yj7-w1o7)PHM~U{TL2(KnL+@x8E!~J zIGqZ~Q_SjS`^4DLo9%FTvmJtY3Y>3mwpTcDH+VK8r??HKh>wQ979d;=PFdA6M&kxK z(X*m!Q5l;el0!FqM{fuCdvH=HU0^rhYCp}-3h4mU?XsiYOO?A)2A%|AzcuOg?+iKo zyhDo5*npD0Y%-)pL0S%qNXmxA-%vEPU$@CmfHjO>X zydkATmEe&*oFrOwx|5Xe!+g7T0B=lAvPHP;24S-;!)M!u(YnH8 zLo?gUMUs6d9?)RlBe6R~dM8G1AV2uEF03=FO>3$8c^OI3m&UuVi^~W`ibwCo zmKf_5O%BX5F`#vZYMP%=fTh?XQ04(0Q-Xt)^$0zl?N>AV`$N7eHrjKvnY;20qzln4!4ai%IxD5$sT3~Gp zjBSIhO_&8^F2#u!5;w?m(h4$hgV!Yn;p5WZ z!R#l0tAoK#eBAZ3 zqC$%1()FHL9opODqYZXcE2cEd5N+j%@Y=SLE?V7~TgV(YD|g(W?6Ix<@vo`&tCdau zX%v6glUd}6_gF!o%$#I2T#Z|%ukde3NK#=kRMhkD*O!wvmX*FJ05+4Q{spxF*E~%( zaFX0p&$8}*{co{5wT-rQS@rmVKcGc+5xZo$9x7`S_o+ z;_}{ZIF()-?6KfjA4+uWQm>}m*6#g4ZL>Cg#?-QMtmd`VY4;bN_@9g?ZaPl? zP@HiC1-=RDdy|y+hN*6Z;?}I!XCS#ZO>TctQX3R8+9`iYy$@clv-w4Gs)MP&{ZYsb z8o~OoG%P<5c-vxrsRYm{)T7|^DXqWQlS{A{^rsN52o^T1nGaP)T5R^2v6zPMS)X zFIfn-eKg7L?j^F^7naYWqFJ;(SjKP}c3Q%%8_7hMt;!eG>Q?&0I=T(c@*hXyY6s8w zt{M7|$u60bn?8l`W%WH^qUkWSG@SW^m_fw=Hc0A`P0eF`F*&7PH>AgS2kTZzXTFH%7cwE7yyMO`J$euZubj;)6Ea&Svfx*LB6H z2e*YE!cF>QY~`P^p{C+GejC5oVhy)itey6YwTUWvYi!=gf8$1cIkxcY*xJYAru;qH zUN8DmOgBKXYdDi^d9IvMhu0W2QkX}&l*pZ`McHl890V}q$kfu?rLE_S-Fcnoi+#=g z=eiUOe(|wf)t}F1Um5F*dh1d^6$b@E6j#tjohvg+blYvXS}PLw%l9{MYFaR5AvT2q zVcUVwqKVFM_0kjqHGFZn0APrJ+t%G?3+vPtR;mrHRhwF^Hnd)CWW}-&wq@H5>{?sc zx3;l!ZEEk@0D~K}X!SllOy?2OpUE^*o9ocD-cu@XE$Z6xzPcf6T2QHr*wkVgO<`em zAGc8+n0w^ee6}iQg5M-1!qm%;b`7zA+Y5IiUyE)0E^Y+CFIVL+b5(x1=Du8Wzuq-> z6O7uga#jAzT$US%qV|l1ShUTQYpABi3(@txO{8+fw8Th1MA-(rki!N zVRNw@3i>B)Ma}s3^Zb&1)$s-hv89f}v3YN?^f%(5w4DZM^M0A-K|(yd0f;w89M3Oc z3$%%)PuvLdz0b8bTXal^!FO`l@C01rB-l_4ZvqPDFtHshc=v7hDfi6~M1_&+JLKv7 zaEl|folBihF_h28c2leK6xm`b-qkabFoB=`C>gmM~z$su+NWU-Qm4F z5WtkHGM_*~Ve+PcEIQynlP6;=^l$_z$@^$XO;&k1iS=ikWtQUOS+btdN}-p`n39&4 zxHD+%FGOFnB#-qrUN0skIGxDNRbDOm3cwO_+!*@BISwLTqJ`k0rMTT7(~~T3K4{9# zX0tZ>yZ26H2IqP1nC@$Q8yLvu-y5oo+B+!=ijS0oyc|R zHHNM?wixyrHmK(PXJg<4$=d2hlqw`^=~}%_ zE&xGxcH(4{iYf0j*MFDve}|d)fndVNksJ*-oX8Re`Vu7I?Z-b$lt3siq}uXLb3nGq z|JXM7;}&@zTO^-qneXvBX{wqqB%bH7Wsb)-8@*9(N1t#F$yI)*E#E}cY}lHb4Si}h z{H3Vb@C!Bb_v4?Nnhn2Dv+Gi`;Xe{Jga7^SNwppwAhj-5wSp7ih=j3^h&t@}OF)cfsied$H;~Aq1j< z+MkF&E61HH8v0Wr;agJ2w64jO)AOfZa)Jt6fht$<-%lN{3XDWnssO0g zp<2(_@=7Rfs|y24tx?I4J@Fmr53HKlXX@tQ?~=dc4(cXQyfM^3EFNp83xzkfbm>96 zyH6qktP@l0zdFv1qZnR=Du*|IVAbpf0uzMYuH8*^NQqscYTKa}s@AssE?8;XZqnku zw`;05PTDE2-q^kkJHe{3x7{UHIhs&2va zXB{DHcJC!==yW>!$iGsBR4nIBalWdEHdM==FTfIo^<$@kgjLd#vpOwGNW3(!uo5=) z_~C;^opG~2d=Td?KNr1y_+T9j@nSt&&DVltjt9w0sQHsnJsL2wNgT$wXP`|9NVw~% z=w?0>phyTG5mxjU)qK1@0t3(_U(ZC3%VfGJ~4jgGOxgJd6z4qu2( znO}$%v`U-aqD|D5+*$edJXsV~pDGqx#i&czhM(7EB{0J&S-tAoHRB)qVU6B(*YOrl zb-#?l(D93SD}s^;0>Nn)fD7;OBtfMeG&_(;W-#XvY~AQbXeRnO}959c_T-6%?dES2nBET!y|jSKT)tOCf!N8pQo zvYxM(-L!U8_8pOjLom>*kFCj3F=T%#5(4LTYgl7n3Xd*;8hJevcl**A2jE!=Lf`Sq z%rfxe+t6w&oDt2=d<1KGxpKFJv1Ymo+S1xB`F5k7Qa#r0QhcT^eg+re>2g_AaJR`Z z*Ru-fBpe_n8NfJJh)0FKtnx^VIJWa#v{R}&zE(o=r%hP;OywhebLkL)% zY=8cKo6NpjBJ7GB+DSiPj< zibx>wTsa+o5>=h5oqeJXE=08g@zC>;w{$bgq!s?fUTG;YdH#Qc8xQ?w4?^U^-@R zZe(7#$v8R8Q6#x-cv?5d1~@+fpjQUc`bqm8jaQe80{S2s0UZ->ZWqyOjD}$S*G)G9 z;l|S{&Vi?cc`5U#r0}OzY1N!!*$0I?|-$~BP;$ow-2Lg3FW7ZciH4#e7+=%JYKo?aFOe zgHv@TDf@N{?m#_qT7CI*zLh-Xt14J8!C!+zDel8*4pbREuvR3PB}#U%@B_d4 zkYFX@;#&#{Su|G{0ZRP9x=U9{3DFK{6G3Q`TFnLo9UadrsQ?Y#NPY4(q+f+zoKn6Jf6w?N9Ba7#Q9 zg5q#~5_N~*QqQ=LJ4(GtX}s^=Rlk`&QTC-8-d5^Qr4s?y%wmG^z=~hx~8sbh!6 z`#j_9fb$;j+RVSVdmbO{oOf>)gmG};x-ebio>5C+7#teq${j;mN^CCGf~g^hDQ zi_4d_tc(S7P@)d(u>GxzC`S7g!V~%ShS21g$~%7sL?Jjo3!ca&*bYrNTy#VFQL~4fryaeMQCIh9rvx^;y zbwP(D)himDQi{=7K|)W`_DjuuIU6MZ1@SprQ47jMDaFFj<3eCDnnRWtXrsE77w!zA zQ%sJ}Y0||6uwV~Z`=hIq?&oh_!mDFl7o7_p(^d04a zB-2gv2kbfVXbeCw#1RqDtFymecEz*`pQtVnakAL%ydIt^U2+kX5g3Klj#sGxYc}paw@JT z6HfUEyP_K_AJ{!L!DEX~!E!8!`v3&WOQT4 z=rfn?Jm^UwT_Qe6r;NKFzZA#WVDYP;fcszEs|?xLw*YfkK( zUl`n|vSD+g=Bb-l7^Dp}Z#v*q-WnkUVVf_V#j=P%>(9;b_IZtWQ0wUmzA)m``Q*G9 zMVfTAcd3L%Yq}a-&YA0-@4xDJukr@-`J`(bv{U!BA_U%xpwTRi`8qedwz~aW5jw_q z)gb3taVdjpY%H{V80}V;!cfoIZev-m#fgf#GDN=j2c}PGCJxl@DV8`EXl3abI6~WP z`I~@SV5qUxv#&w5zNk~kQU`f;;-vY3v|Q4$nEL4&`q9$?THh+(R{U5`A#=y1Jeg(p zIY#+C$;~p|yqs;`iQKX4b;2m!btlD%-wgm|#g*LzXm(`NwE(Qcr%igM`>9U-@3*%6 z3T&@J9s|NI!eAFl(d@mn{EPk9l3vGNoCP{`bb$2K_f(FxkuHI;WvlPPGe*W*)GfK| z@4^}!t*Z)l;AlP0;rk+=b{(=$(F1}WIrIgQk`PJ=jMGI)RIA&;X?J?webGzCm$Q6Y z9$~D8jiG^VG@%z=g^0fHZZs~Z%dET8OB?%CUShq#ebLlkbiV955gmKiL(R17yOc*A zSJJ&kw&|2pq)sn(n{Cmvoypog?tM`5ppQ-#82sze6&6+E$Mk#ROg^8 z&Sxjk(W{>l$yb%2T@>JepnsU3Q8_7r!D0rx{pZQE^#QndUqT|H%@^1sxMGp(IEXe2 z?T*2C-t&7qV)sxaYbhMD9rquhc!uH}&1bIEm_ayxFhmSg6ttH`L_ z&*h>x5|or;ESv&{-f3bGBo&b_9Xir{2%Hv+xu61rkkh3_+qiec~Nwjy5jVw`QIQKfb_p`GFiw`U6X!3VqSIONNk(a=bcCMKf|EO7>-}kK9BV`as@e^BD!ffIAOXWR$99agm8h35=5J zF}yBeAP8*1a1EFS6OQ}15iE<{A{TUZki5;uV_x}CT)fp>95-n4;X{8Yz?|h`AW$<8 zt{G^Bx^9UIR~m)VX0kE9T4u{e%Nn8S_QZ7g*Y^Sij0}We19|2AjC$ z2SyJDAh7Nm?O=^DxbC2y~KxP z);SDcS!1}nee)3*!T1Xo>x?@aQk7V%;d_6n)b~+W^G2*u@FRRzZoi+BF_mB?k7b5p z)IRJq{+WEjJVW$$USS;ZukRFgBPRGpx3C5q{1?Zt>)=YQYuI(L<@5NSH2H?L!ub%sy`nz)~8-LlE^&p2H*1N2FHy$3I%7ay?i{9 zdMzNNe^1o4C-7}BD8vFb7;h5l$qJu9Yzp29jX=aAHR6Nq_?FI)!L;6&VN-{IZF~Se zyU{;l+XQd$zO4oRAHyX6%UZN=#2Ed>qW!B`w1WYA5yIH>5`(j0?RO3g zy|C@Xva&{KYn>t7e@7gI_jsq(qhYH@VYBv5Ygk&DuiJsL^SLe3G93EaCh0Hc<9{&o z@rD*ves+ybHAY6Zu&QdN;(T~rn`(Qu<2P;3)Oz{rTQk)u|Lq$sZ9MyDH(MH3Tt(N7 z&bsX1Jmnx}9Obzyos^?O3}jyhlX_i%^tBumv-7eNh;~w}x{xXK#SaHBU%wc=8ol`b zCtI$BPr@$Se5fq3dXb59Yx4%Is>@J1eXCPc%I6oU>zZ&>ELKcf1#cJeuuaHzI2-qI z=gM7$Pg>f|X8=cWQdG)}lV5yQeF~^AQYuF!K#>BNHJNGtu?rJoPtzVwQF+(d$kXL5 zb)f&56X<`I@_>GYRnwvZ%fE3`F@{RVHg3pX$foTz3L3J1En7LwQd@i*zSz{r{T=j9 znFe*>E0)C~11mHAqWu`gvMk~QZIG3vFZnVXMVz;`^lF3DS#IAl&dlkSspT+JW zuEPX9?qnqG?BVXL#}>-Inc+ZuqMkI=t?--1zYgLf>$J z-EVx0;e-3zoZI+h9e*i-G&F}GBpU*M?E~wAz1fkK#!wfmO%0*JwTl`-`B`?ptfLdO z0nBSS!s4(~>Q!42r_ZX$GRej8ORwz85}h!;Ixm8xWnD{Eputahfs|{xcac`58)Bt; z$&e2jaIt91Mkqe4k&mzhg|>`@{n5>G5-~rpn_?w4@e^}C2{04Zs~h!x^VkVHhlZjC)AW@e7vnf8%UR9G(tC3jVE4)=UU}- zA{xx?wJ|S@<^>}SC~2p3K{>o6p|~%7yCB-kfPg+*Gu#n)V^jx&YZnHw_v4o?c~Fz7 zUu7@CecdXJKPZtXd;U#iy8kU@Fui7i()#42HJz~ zIf>-iS+JZbo|+8*8?aPn@et6gHWV~aNHR6a8~4!}0O%VlA4I;dlxfjQ+agxT&W=r`KQOT zl{yMaHFQ%{MT@az>`c3n=j&DDbZ>U9^YyBEmNz*?S>M>ybm=-pUBW4kAZQMeW>k(p zLU!~jpPdw3v&~2YHAFxT3&ACTc3FRWJ4|bw?*Jr0`^s6@>&zo!JufA6>h`Is^LU*( zm!J7UUcy}mroPuj8tOwQ`s6dWOA?4CliY8?2yR~`EDyNJy2tZf*Ytuk{g27+d-rSC z*OeWyhVSuU5NEGd_{Vw*Akg{{nGXHFO4*kdyQ7^c_+q_%_WMVkl*~RjpZO~Cj0qX8 ze6LPVfKAF?G_!l=o|0Z*u~asj%tdpjv7eJBV<#=xm~tRw=6FSC)22^;9x}^av?di^ zR#&pUt`@YWgOckp=b*>&_z%DT-IqUK{(SlK<(BoW)#-G205}8y DWXKtM literal 0 HcmV?d00001 diff --git a/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py b/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py index 46eb33f8..ad34d81a 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py +++ b/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py @@ -75,6 +75,34 @@ def _normalize_b12x_bytes(root: Path) -> None: path.write_bytes(data.replace(b"\n", b"\r\n")) +def _apply_b12x_overrides(root: Path, archive: Path, contract: dict) -> None: + """Apply checksum-bound selector files only over their expected source bytes.""" + if _sha256(archive) != contract["archive_sha256"]: + raise ValueError("B12X override archive hash mismatch") + entries = contract["files"] + expected = {name for name, _, _ in entries} + if len(expected) != len(entries): + raise ValueError("Duplicate B12X override path") + with tarfile.open(archive) as source: + members = source.getmembers() + if (len(members) != len(expected) or {item.name for item in members} != expected + or any(not item.isfile() for item in members)): + raise ValueError("B12X override archive has an unexpected file set") + replacements = {} + for name, base_hash, result_hash in entries: + path = root / name + if not name.startswith("b12x/") or not path.resolve().is_relative_to((root / "b12x").resolve()): + raise ValueError("B12X override escapes package directory") + if _sha256(path) != base_hash: + raise ValueError(f"B12X override base hash mismatch: {name}") + data = source.extractfile(name).read() + if hashlib.sha256(data).hexdigest() != result_hash: + raise ValueError(f"B12X override result hash mismatch: {name}") + replacements[path] = data + for path, data in replacements.items(): + path.write_bytes(data) + + def prepare(destination: Path, cache: Path | None = None) -> Path: """Create a complete, checksum-bound context for a network-disabled build.""" destination = destination.resolve() @@ -110,6 +138,11 @@ def prepare(destination: Path, cache: Path | None = None) -> Path: shutil.move(str(source_root), b12x_destination) shutil.rmtree(unpack) _normalize_b12x_bytes(b12x_destination) + if b12x.get("overrides"): + overrides = b12x["overrides"] + archive = destination / overrides["archive"] + shutil.copy2(HERE / overrides["archive"], archive) + _apply_b12x_overrides(b12x_destination, archive, overrides) b12x_files = _package_map(b12x_destination, "b12x") if _map_sha256(b12x_files) != b12x["package_files_sha256"]: raise ValueError("normalized B12X package does not match source-lock.json") diff --git a/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json b/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json index 7ebb33a4..b01e3efe 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json +++ b/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json @@ -6,35 +6,171 @@ "donor_revision": "3512b066e7796128c0c380ccc558182960f2f0ea", "dense_donor_revision": "a8c796f3af74106b2d8d441e9ec54588936a5388", "patch": "vllm-e02-to-compute.patch", - "patch_sha256": "a41df4b7a6f2ab4c73157349cb2c66e9d89ccff05af124eb66783b849ae98e8a", + "patch_sha256": "4c20d5874a28cdadea24b8f1deb8799e950786aa39a6e1e866e0f67a1ad6e297", "replacement_archive": "vllm-compute-files.tar.gz", - "replacement_archive_sha256": "cefc8e7924404280e8ae024392d326fac3262a77d260af514d981921fdc93f90", + "replacement_archive_sha256": "520db578e6be4466d46cdda7698f4044853111d5810ce6da58f81d5a16b6ad88", "files": [ - ["vllm/envs.py", "22069819122e630dd5131627c6bb8752820be2b66178d7f445eeb6ce93e03d32", "1fbac28f1a763f9d27845a8e09d59f71ba1740b4491a74d0097a7e9d63763804"], - ["vllm/model_executor/kernels/linear/mxfp8/b12x.py", "7abc42bccf03114e880871fa2ffd67d11466483b2ea636d466e762c17417f3d9", "1b4448f7dbaadcf9ef93b59a6aa54a016653a1019ffa31605722d828ea3d80af"], - ["vllm/model_executor/kernels/linear/nvfp4/b12x.py", "4c59ca2067e78a731598a47d146b9e8ddb1b8270f54f6ab19b72c30561f9d53d", "7a7036e6d0254a0ee6e3265b18685128fc42cb1b896efffdf27c9de038d69fc4"], - ["vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py", "efb07f928463fcf12a3c079f4da2ffa330d697a98a6afca3bea94a0a21825f58", "4a83c1eb9ea9c47c34e8a87b07f087a4cf1384428e74d800e0bdbd667abbb63e"], - ["vllm/model_executor/layers/quantization/online/nvfp4.py", "52ce69a32611d8eb00fa2a27cb9b2ec46b8836f34f0a1f8b4834990c3ea228a9", "8f85d00e6ef0effa873605e506c5c678228cba1459c2245a0770e2a3d0f74450"], - ["vllm/model_executor/layers/vocab_parallel_embedding.py", "e86d37bc50b5171e2bc02a8e6428feed34309807f63d20bead6bc5db4ac7fddf", "0f958f55912dd1f6bf1f902110404018263afd9c0aa34ebb66236780c73b9243"], - ["vllm/model_executor/models/deepseek_mtp.py", "e9270724c39a0152dc0a66b94622ebddd384c592534cbbf38d2f43c0ba1592d0", "ed5b247d014207e81d7d2ddf88e62e82afba70427798428b1b770a4f687ee869"], - ["vllm/models/glm5next/model_state.py", "8c66d55da1bcad63f703b1a5463b6969dfa1d3308d3d14abfea778f6c5e34b7f", "4f884f713335d55fafd729fa4cc9bcd88389ba005323eb79a3f9b02cf5662f36"], - ["vllm/models/glm5next/nvidia/mtp.py", "cbff653af56b3589a1ff3d52b5fea660e718cbdb1b278c4fa5dcd8eac0908ee9", "71e88e3a25d829d52d90a394f7306e29759d3951fb8f912d1279fa0f30496099"], - ["vllm/utils/b12x.py", "f610dc19b4dc10d27361b075ff7e8a97a63244f09f770dfc2ff04aeceed4dce1", "63e7fef8c75f5cd01678b338b9154fc1204b875794d0356904c461f4e9db3076"], - ["vllm/v1/attention/backends/gdn_attn.py", "7c325bbcb612aacd2411b3305ee6d068b013f8b6c4445eaea59059ac45881a3d", "fd75fb72efeb762ec558d332364e889df44a9ea483026bd5ded8f77f42be9b7f"], - ["vllm/v1/worker/gpu/attn_utils.py", "012399dde8910deec550df260c132338d20e20543acc7433ace60fec1229814b", "80108d185a996e52466f2348d6157398248acfd05b735e4a5792949034b70b9d"], - ["vllm/v1/worker/gpu/model_states/mamba_hybrid.py", "f2bc9ef85896df2508557bfbbcf2c682f82414595e8a562efdee6ed3dd0515c2", "b34cb130e233f4d322390acf7fec01a3a368090c7f79da1fa7770c622a2f0dde"], - ["vllm/v1/worker/mamba_utils.py", "b5bcf2c170daefb858b0668b032dd252a379c78332efc91d87c34d7fed2e9373", "826f30b8719f0c715e9743a71dfe48f2ab60f7e0de97e2d542ccbe867b6f68d3"] - ] + [ + "vllm/envs.py", + "22069819122e630dd5131627c6bb8752820be2b66178d7f445eeb6ce93e03d32", + "1fbac28f1a763f9d27845a8e09d59f71ba1740b4491a74d0097a7e9d63763804" + ], + [ + "vllm/model_executor/kernels/linear/mxfp8/b12x.py", + "7abc42bccf03114e880871fa2ffd67d11466483b2ea636d466e762c17417f3d9", + "1b4448f7dbaadcf9ef93b59a6aa54a016653a1019ffa31605722d828ea3d80af" + ], + [ + "vllm/model_executor/kernels/linear/nvfp4/b12x.py", + "4c59ca2067e78a731598a47d146b9e8ddb1b8270f54f6ab19b72c30561f9d53d", + "7a7036e6d0254a0ee6e3265b18685128fc42cb1b896efffdf27c9de038d69fc4" + ], + [ + "vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py", + "efb07f928463fcf12a3c079f4da2ffa330d697a98a6afca3bea94a0a21825f58", + "4a83c1eb9ea9c47c34e8a87b07f087a4cf1384428e74d800e0bdbd667abbb63e" + ], + [ + "vllm/model_executor/layers/quantization/online/nvfp4.py", + "52ce69a32611d8eb00fa2a27cb9b2ec46b8836f34f0a1f8b4834990c3ea228a9", + "8f85d00e6ef0effa873605e506c5c678228cba1459c2245a0770e2a3d0f74450" + ], + [ + "vllm/model_executor/layers/vocab_parallel_embedding.py", + "e86d37bc50b5171e2bc02a8e6428feed34309807f63d20bead6bc5db4ac7fddf", + "0f958f55912dd1f6bf1f902110404018263afd9c0aa34ebb66236780c73b9243" + ], + [ + "vllm/model_executor/models/deepseek_mtp.py", + "e9270724c39a0152dc0a66b94622ebddd384c592534cbbf38d2f43c0ba1592d0", + "ed5b247d014207e81d7d2ddf88e62e82afba70427798428b1b770a4f687ee869" + ], + [ + "vllm/models/glm5next/model_state.py", + "8c66d55da1bcad63f703b1a5463b6969dfa1d3308d3d14abfea778f6c5e34b7f", + "4f884f713335d55fafd729fa4cc9bcd88389ba005323eb79a3f9b02cf5662f36" + ], + [ + "vllm/models/glm5next/nvidia/model.py", + "e992bd796cc8efe0c55656b3f6d858cc2b4737d100d015cb49556d1d53de2288", + "6ffed6a97bf47af392c7eb89b90e384c80a38a3913794876b1827a1b7f3dec69" + ], + [ + "vllm/models/glm5next/nvidia/mtp.py", + "cbff653af56b3589a1ff3d52b5fea660e718cbdb1b278c4fa5dcd8eac0908ee9", + "71e88e3a25d829d52d90a394f7306e29759d3951fb8f912d1279fa0f30496099" + ], + [ + "vllm/utils/b12x.py", + "f610dc19b4dc10d27361b075ff7e8a97a63244f09f770dfc2ff04aeceed4dce1", + "63e7fef8c75f5cd01678b338b9154fc1204b875794d0356904c461f4e9db3076" + ], + [ + "vllm/v1/attention/backends/gdn_attn.py", + "7c325bbcb612aacd2411b3305ee6d068b013f8b6c4445eaea59059ac45881a3d", + "fd75fb72efeb762ec558d332364e889df44a9ea483026bd5ded8f77f42be9b7f" + ], + [ + "vllm/v1/worker/gpu/attn_utils.py", + "012399dde8910deec550df260c132338d20e20543acc7433ace60fec1229814b", + "80108d185a996e52466f2348d6157398248acfd05b735e4a5792949034b70b9d" + ], + [ + "vllm/v1/worker/gpu/model_states/mamba_hybrid.py", + "f2bc9ef85896df2508557bfbbcf2c682f82414595e8a562efdee6ed3dd0515c2", + "b34cb130e233f4d322390acf7fec01a3a368090c7f79da1fa7770c622a2f0dde" + ], + [ + "vllm/v1/worker/gpu/sample/gumbel.py", + "27467f5b570dc1fc2354ca09fbb7cc6a38a1113c81eb99150b94d0c678a2f13c", + "f22d7b5b6b46666ec6d746e49798ba76cb329407cc749a382639dc8ab83f9741" + ], + [ + "vllm/v1/worker/gpu/sample/sampler.py", + "e40830547705a6622fa7c4733791c261c0908245d376514d7ee891e587357cad", + "3a1382ac5d5e0798e1128854b10d4801f1766eb99e90e16e4efa1e8033ef2fa1" + ], + [ + "vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py", + "c2c1f583ea1572c62c2194af70494d44f5843adbc3f94f204b9ac32016097c19", + "d91dc325f52a6999d3c9a204262f8ca200cebdc568c9e9924b76c78ede8aef21" + ], + [ + "vllm/v1/worker/gpu/spec_decode/dflash/speculator.py", + "0f3d16fb90f30ee2fe45c744dac99384ebf081663664bc8464403ddf644203d5", + "9fcb9ddc718656ede02c856c4b224a1a4511e5d955c96f1e25207e4b87a940f4" + ], + [ + "vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py", + "1f6ff5ca9c8f38ff417aafd43bfa3116b5387bf0f7b58721acb2185781879836", + "a140bafbbe25cf689bdae0362232ab103adee0baca374c6d47f7af1789bc312c" + ], + [ + "vllm/v1/worker/gpu/spec_decode/dspark/speculator.py", + "a72c6d0dbf37ed41b901c0a2178de51aca2c74e6dffdffb89ce223bae2f64598", + "5d11c9570897f57069fa0f6325becd1ff393c7bf2526898c1e40b6502ff67e35" + ], + [ + "vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py", + "0dfc31a7e7f0b6247ce8ea7f580dc930828a885b28c9e4a2d4e69f7ba6b516d7", + "6e25b72630c7f87e7a2d8b64d35f2b7569d5d479ba007d4d2b0deb4c72860353" + ], + [ + "vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py", + "68b60488866ab87857197560bd42ba26c2a89da70d539e9ab56f18667da8998c", + "8f411ed35dc0691d9a71a3ae1b9aff5e3f1d4085e4235e6eefd11e8515dc5a66" + ], + [ + "vllm/v1/worker/gpu/spec_decode/speculator.py", + "2a24c693204032b77bf8abe08df916ca20b119cfe60fd91ad33e87920ae37685", + "3a3574b57748be2e4cdab694c4d5dda98fb2107174b34db4cb1db1ee34e2f457" + ], + [ + "vllm/v1/worker/mamba_utils.py", + "b5bcf2c170daefb858b0668b032dd252a379c78332efc91d87c34d7fed2e9373", + "826f30b8719f0c715e9743a71dfe48f2ab60f7e0de97e2d542ccbe867b6f68d3" + ] + ], + "loader_donor_revision": "17e341b9ede04269f81fcac69a29951a0668a94a", + "rng_donor_revision": "44e6766e3397e8fe8ed9c1fa8a8d2783bb4a2ae8", + "rng_donor_pr": "https://github.com/local-inference-lab/vllm/pull/653" }, "b12x": { "repository": "https://github.com/local-inference-lab/b12x", - "revision": "b58f34eaf978277621efced6678e6713fd7122e4", - "tree": "7637fe5fb4d88882e0d18cdacc68c493f478499d", - "archive_url": "https://github.com/local-inference-lab/b12x/archive/b58f34eaf978277621efced6678e6713fd7122e4.tar.gz", - "archive_sha256": "8cfd2d8bf09169d00a669f72f179c54bdfc49af35b05214a5e5f0fdb1d5779d8", - "package_files_sha256": "8e5418509749db21c5cec933ac5c476fae4f0fb89355987390dc0a424261ade6", + "revision": "ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301", + "tree": "dcf039e5e754136275835ea997e6b9abbb6b15ae", + "archive_url": "https://github.com/local-inference-lab/b12x/archive/ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301.tar.gz", + "archive_sha256": "029a6047d80f759e964eb302803df3dc5d2ee324b0a0725d38694b5057ed69a6", + "package_files_sha256": "9c33a4ef28eb83c81af446e3c02672b7e3318dc434f16a54f304fccf7c2de0f4", "installed_text_line_endings": "crlf", - "normalized_text_suffixes": [".c", ".py"] + "normalized_text_suffixes": [ + ".c", + ".py", + ".md" + ], + "overrides": { + "archive": "b12x-selector-files.tar.gz", + "archive_sha256": "31c51e4fc5b334c9e28c469c4b6f50f4a34458bdd39380c6145dc656056ea42f", + "donor_revision": "9ac142824b4edb750892a0fb63d914230086495d", + "donor_pr": "https://github.com/local-inference-lab/b12x/pull/316", + "files": [ + [ + "b12x/attention/dsa_indexer/fused_indexer.py", + "37e948af8b3ee4755fe01f9b853fc95b04afd3bf31a5cc3347f112e3f993aa87", + "893fbcade135b7e1d146b8fb6530cde0650be515f69bf9a17ced0a9c61a141e2" + ], + [ + "b12x/attention/dsa_indexer/paged.py", + "8ceda277ab3ed6d5154b23765f1a8719240bead889c742881413380f006334c9", + "21a037890fb08896d485de71d1f21eb5cedc6540f4063f0fc0c44151e9701249" + ], + [ + "b12x/attention/dsa_indexer/tiled_topk.py", + "1da3541a53db5dec2c724aab13ba79fa5ad5213627b4fc04f4c9b3a1aa8a98b5", + "52fac27a905929131642a085bbb013989dfe1d6eac9b3a39f576dc6f1841cbde" + ] + ] + } }, "cuda": { "version": "13.3", diff --git a/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py b/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py index 1d132cd4..8ec201c6 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py +++ b/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py @@ -1,4 +1,5 @@ import hashlib +import io import importlib.util import json import tarfile @@ -22,6 +23,43 @@ def _module(name: str): verify_compute = _module("verify_compute") +@pytest.mark.parametrize("fault", [None, "base", "result", "archive", "extra", "duplicate"]) +def test_b12x_selector_overrides_fail_closed(tmp_path: Path, fault: str | None) -> None: + root = tmp_path / "source" + path = root / "b12x" / "selector.py" + path.parent.mkdir(parents=True) + path.write_bytes(b"base") + archive = tmp_path / "selector.tar.gz" + with tarfile.open(archive, "w:gz") as output: + names = ["b12x/selector.py"] + if fault == "extra": + names.append("b12x/unexpected.py") + for name in names: + member = tarfile.TarInfo(name) + member.size = len(b"result") + output.addfile(member, io.BytesIO(b"result")) + contract = { + "archive_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + "files": [["b12x/selector.py", hashlib.sha256(b"base").hexdigest(), + hashlib.sha256(b"result").hexdigest()]], + } + if fault == "base": + contract["files"][0][1] = "wrong" + elif fault == "result": + contract["files"][0][2] = "wrong" + elif fault == "archive": + contract["archive_sha256"] = "wrong" + elif fault == "duplicate": + contract["files"].append(contract["files"][0]) + if fault: + with pytest.raises(ValueError): + prepare_compute_source._apply_b12x_overrides(root, archive, contract) + assert path.read_bytes() == b"base" + else: + prepare_compute_source._apply_b12x_overrides(root, archive, contract) + assert path.read_bytes() == b"result" + + def test_source_lock_binds_patch_routes_and_environment() -> None: lock = json.loads((HERE / "source-lock.json").read_text()) patch = HERE / lock["vllm"]["patch"] @@ -33,10 +71,13 @@ def test_source_lock_binds_patch_routes_and_environment() -> None: "replacement_archive_sha256" ] files = lock["vllm"]["files"] - assert len(files) == 14 + assert len(files) == 24 assert len({entry[0] for entry in files}) == len(files) assert all(base != result for _, base, result in files) - assert lock["b12x"]["revision"] == "b58f34eaf978277621efced6678e6713fd7122e4" + assert lock["b12x"]["revision"] == "ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301" + assert len(lock["b12x"]["overrides"]["files"]) == 3 + assert lock["vllm"]["loader_donor_revision"] == "17e341b9ede04269f81fcac69a29951a0668a94a" + assert lock["vllm"]["rng_donor_revision"] == "44e6766e3397e8fe8ed9c1fa8a8d2783bb4a2ae8" assert lock["environment"] == { "VLLM_B12X_DENSE_ACTIVATION_MODE": "auto", "VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH": "1", diff --git a/runtime/glm53-spark-mtp3-mesh/compute/vllm-compute-files.tar.gz b/runtime/glm53-spark-mtp3-mesh/compute/vllm-compute-files.tar.gz index f0fa7b41468db0e226ba5915630e18a17b47c4b9..12aab7ee8a72384dbb49661a42f002af07ee3923 100644 GIT binary patch delta 96303 zcmV($K;ysXi3ZrI39z2tfBkYB$I&;Mzfr||*g%z&g$=AoS$0x%;0moT>sXQ*OLp=c z7HhQxmgI&27RCak7?s{Sf4vv**15vxO3rlm%@G;kcJkPjiiq8r-udk5?& z%CfqO4Eb#V{$dNTeWi)dG1_#Wuh_J7d0U2F(9G+tXdUhAC3cK5h+jgjyJIhRY^s%f z%NxSqC>^c6=5ZHmTL0;Ghv`;U2MD!sEe>4N8mr%j##c)#(zK}>1{KYQMt|UhE`BoU z-BBesVL*m+zc@IKf0>RzGlv14+?S`Axf?c&jw`V_Eect{Yy~pBAiJ%h8?T4kPMLnD z7RgQL&4b|>^P*|&pGD+TYv`+Zhc@KjdAckgri)pd@zXNFTfp7J(SqL`#zuze@V)TQ ztP$InI9sd5oo{(;tqUE{$Bia}ppklPDVB{|4t}Q}LXLVZLxl8~<&)_AS`1I^QtsOE zA%3dNQxn3@)o+~luJ2~OO+-1Zl^Y}9!4Ob8dy=!v*{a+uJu>vzvjkY*HPKb{+xcwC zlU?8?f8t~Q)ySX;n^1mewjA1C8=R1351XzTh&HolAjy_Xz?aaq1`X8$RgzqC6W0;} zp-dBR#Tv~W`1g#QvzB!JfmpU~K3yDAD7##oQbQXQ?hYr+_B2rS@UD9_e zTS_5sD-pA+WLf(<$pjptFyjFglrw);*_6H7f592JrBHWMS9>!LfYBAeXvrZZuaub5NA8N@xVn(>@6jIoGD=x@wB%AKopr-wd3BMRB%twE z7ko##qV{jMhs{%~P?p%XT6eFkw=x9ec5rHL$Az9;EZ3=CL+B$pHrEtWdRksB%+mTd zTUstSSynGUCccY~^sz1R34w2|&)dqbfAwh)sX|w`j_WFDC~FY|0{9yaYRzC6xIO2c zRwZ}=|B{4kKLSyRiz*n2L}pQVY6bQe*uIT zxf9|lip3^pPAerZcW?rZP46E3lrIdQH7LAVntp~>&o{$eWhm~BrT?c`enD@?_90m|}XitRc zQz5Q)lP7tTfw`V^`<;e`Cr9(hfAl(6=xP|9#vcgGmxK;~Q7ofHxfsbh>Xs03%4z1K zSp9J>p5o0Q1veCbi}+a%5ncaSdQv5uL2^?1Hj?^UHx)i6oWO zOoTGx7>mT%Yg?}s%DL0R61S#`*>YfMiNRQ$Tc=@k6G0u3VWV~BMB zcB?XQei+$=1eGJ8e(V>uqPA2OJKDRYk^*~qcGH1XSg}y9Sn?sh2Ej<4t zND@(UR!oYuWO>R%Eat3Z4@&^h+*mO)#zP3HkEU-#Xmhq^LklHKD;-iscCE;BC{ zxtx~lomj0pT}ru(?8IMoWmp*ZH%kb|o^9p}<%kyj_y7Jse~KJt0|Si5-4GGbY4@Y= z%d5uffev6X3GUk9@~{_vg-5g2tK|yU$Fp8y4v^|s9{PBj%G%(SNPZ7%bn@mf0d*&j z`MUCIf=K+qS|{jZ-!L$z{nzI1R7Wgy@jcS}5cQjC!|7mwL>V7{U;q7~ztv!1C51yt z-MW9@Ak7^ye{k6Svv%yP3x_;c=p)7%%NLVJ%eutV#hoFpCAZz@#t6(PoD2Vr{~3K^ zjEtDpeom<(0N4Qy@qzQkHC@9)Z?ZtfttSFtzu!`*=L8*{&DkWyxphvhf%jq z3hCf_A5ghqKiBP|Gg@8wPl>K%q)AV}c3e>p472w>ph?ligrP@_+TAs8>$s!tX79D;0(x_p|zJ)Z?hP4~Cf#_E1Kebh4+`jU7 z<3+vBe_D1^lOj-$=E36nD!Z-|OY7}oLKaHAA!xx#-b4-jx93Y~G{8*^X?@VyRbdy+ z#N;mG=vPEZ zCxs5M$giTyY*i2#O{k`=qSsS(BSpPUYm*;pe`JJz&(-89bXY@sCm(usQMj3a*Hu)m zn&=_!K3JG&bhR-FOieJtXo6C_2X<-`F7kn7(Ow)(GY{&LPT2lUY&RT=5k+?>8Ejg+Jzj?QV(F%vGIjLbA850((&su z6S&D;TiOcz|qK%Zy*;X41C7%RIVm7;`KnFpW&-UubcO@{K3c`OR z&neb-aT?y-g!eH_dE2xdv6RGcDhV`*f3OrAXAgb7dDGd1!#9K7%&gAFvWT|`D$3vk z3q^Srj(+?sdXrDi7vk8MMc?P(wmZb&vN`(YR!tO4|flZl(mQsV_b4`sT$}19QEDWwwO{Ido(zlUbLuQMsJRf7<;a zk1BSf1Et+q?cb5#_+a5VzLVzYM}ln2iO*`RQV_Rfsu$ox7#$AZn}D3H11bGf8?-pc z3qmGhbi5$!3tU)1wgbgoIvrTTZlmi#L+F;(GaSacDi`IJHbfaW@4n$Sg;j(x6$r)| zGnvwB5^@V$J;Yt!iq{u&8=G)se}4oQTtblzXuf?|Kn7w|1n8GFH;mrv7JkqxvLB)) z-3us+-M9uv2>kjtY6^Ij9;9@V=Zi?9h(U@Yd6OcPj@d>heyW3rlQ1Q)Vr1sWadQzN z!TmK2)w^Mslvq^~%$gmsgb#jaVoi)a<{(A~{8K2jR6ivsck*MVLT55sF|f*a@C;l!sS_m};PXTGKVHgXuIfFo{m-x(Uu&^na+4SLLwQ#i0H4&_Ko! z3w)K6O^z(!)?`2nPcTR}nhD`?7K*Dj2FzeUmG1#*$H}|i=ZI?KJw0ciN*o&Rxej4Oz`s~eW%xz5Tz^`uB5SCEnM~`;hd9jj!t%iGBmr z862zTa0R+6cb(B;z1eCUlD6)(bVk1I`@Zxfu>S71|9+YOYW(-=fBxBQ{y9XfNj!MU z?{XX9_fGilpZ#g?^9K(8`@=6D{OLFR_g@YE`<=kQ_r!7i5*F5IwOK3#uZe=Rd=Q>YhWhSmt);?dXe zBv@-cPx1u@d{P%x)hF~2dl#-;^SUaTXY-RRJ-Cc*pFI+bD);@GYInS^?HA;7cd2zKNCzD_J#S?IMABGzD{74uSP3c3UZ%` zC0T9egrTJNf7&0b=w$D~JH54Nu5!7>q|%FTXn;~4+QyCz-ZTWgK2MqN(G`#pATN?e!6P2)iz zJTTeE!;>`aG=E|pDehUIM6|x93m{iWQH+yhWz9Cbq)Z6Ge$-$q(8&F$Mi{A8JM*;L zx6U~gUT$QV0z_<`z>n~;J10J{a5=C+L;LGP`GJ9t$64D$4aAW(a1#1yV-jqH5SP-h zYEENZ6>v!C5}AwLboP1zy2^`Du;VyD^#aP(p1OmcVSkx_bDmd}s{OQhmrqBNOo+#@ zF2vNI!PckHzzQv$#VX3CQ?bE7GKCv1ihw$q$u zt5s{o6n|DRV|F(i8*CbQblkpW+-Yzp&AZ*mk!boWZJZuFq4CU%H8hTCX)5IHUsY@E zMR<43yPJh*OBQ2G|N3Q`R*Qg?O~h>>ed1ra;A?l^!2V>p>EEnr*35pS<^{%2iv!|3 zNF}S6mpS5>kw;(%{)JE?1<`un(dC)5nh>$lS%1DbTc68w%_M=BkP`4i=ce=^-q~Xp z&L5lnAA$$!@4})XC7>tYp7(AOwC~vQ;ciuDV8N@^ zm461lFSLX)B0~U32oNd@SD2EXLvEE_jZh8|l8wB8hS)i<8(n5I@nCX}A}<9AV_=X* zyD!9>@@kQ<5`Tv^@ZBRl9>)o6+BAlXok0VQ)|193xo2!g@nNA~Yoi#pi0~)EQZZZS zCNtJH8h14WOuvCEE&(Ci{(MMDF${A1hLhv(Dt||N1G_!!L@J`r?5NSqChJW$!>=G$ z*AtMak6Sh#yN12az$K@~&M`?DZ>UoSjbnSsLI32j+cnPDAbxDiP|z6hn9trnA2#x zgK*chM$E_QQbrU;D6RNKLdjkxTB#*TUlOYl$pqE>{sJxZx9kx z<q09vlfl0l{X6VhjMr$%U-s&p^97`Kr%Mn`Tpi$0I*Gtdbq*&+k z*fhBR1obOuO)8{UF+wLoe49E1L&ePbHkZ4d=_7+P>L=MwEp5_uvUu1q7j9L)73Q&O z^|H%tOtM%S_0<;bT$5cRKql@5H9?E-E|$zah{p2&w9Dfb`8nAmBV{Cd+89eLp25sT~eZ0)WhR#!!f{9q;m1*7^yH49pqYsms{v-C3R(yw8?DVDhZpxn9Vt4!RP#Y#DUig zZh{s|h^<(X#J{j^NQoO_aZ-$&n2RNOm%5ZtEm>1?6B4XOIHIG)Cx zIB?_#zx`>n$A4%6Y#@>xSxBW0Z3&VTi+2$HUvcncxM+m73PtIJoZAQXTVawwC+H^`CYTSEv`V*0@3FZ>|h$f_XOdq-d=w zjICepn9~j&w{KfRG9Md%+l9>iHx6$`OHChJzwCm}h95UFfRMH@4!v-EfhNlomEn{f z75}v}K7SO80^0UzF|n83*k(CH7luErRc9&0jSYd$F21U+vb1)Ou^YJ*XTWmpw9fZ( zSmkK#uBv%hBybAipQ|aR7>aqW&3CVrsT(~F_u9YlvSN~I<5XF#Wdh0;=ZJ1MTovYF zn|g65-kq1@!PJ<5E9Al5>et0wd+awP$TK3`MM17GM%(JHNZti0=x?EWHOE%Mn> z%zSn@I1b2&P1oReW}x2Q`#`cVdrbBQZGx9?p}Afr+B z`uq0m3QA(f6!XnIyla^6`r~GQLnv+B47W3)hWWk0RC;^r=bhVnxVp`Z+148mdMdQI zS(+kT38(Y`*n=N@)j$3X;^E&1|6y@iOn-}v9^U}~L?iyg-sgLJpMCD(Km6%8{D=QE z{0EALpOiCTR*<(f$xbFZ%wJ{^J>_&bI;o0rW1_hVNg#ekFdk zSH#9RAQ1WLET7sbR?J2lsZ-SB4u37gT)e52c1!)q7x?^8J#M9gR~FK7*20)D04(q< zhAsPavzQD6&mUzoyy8)MY+d3p+&_e0zlIzs=Ii#(&(9}LGrxI+>*G);0EesmJGrIA zV|ZfztY4l_Qe-}2J?RHx!DL{aX>Wpr7_?UY-;8G-Z)hh&7$3>}9ps7R=6~WJPY?f` zzJ7G@;z|1W&3_#}85)m&Jb3Yg^;qT}O^MvPOixbtzUb1k%W{&P)GVV$xwRUjSM1Ag z0Ph~3{RlPHizljH<2U8aJVL1|ICZA$EO;w@G>BK+{@p5>%9Qfy6nHFf5X^_M zO7Zrrm5>mE9;SPjcB_Rz`+xBA@n$~1Mn|OTYZEd9Up_5YVxde6N~HG;{9JG;bZNUr zX<i_6pfK0DlPd)NM$ej8y0H zRVzWTe2JZEEDf?)Ds4?DcZ*40FKz;vtv7^b<{l;yHYylQp_tEF|Y%15H<8&O3p z$o1NIjDY#h2O-m%_-dxu0}g!lS{99jkBW3jrPFHTGQ z-H!!;H@~TeNY7#-O{OaKv(wf?eEXbU_Igs-8k=%foFdOkQ-7f>qmTcSf4oT#4<3E{ z<#oDy zRl1Ow%f(mDPP3D?kBGlc^k1hc38y0FW#=SnSxry9Nx{7^o&S_U-4DH9Do*Y6?@yjT z`{s?<3$G4dK7V2Yxv`L#%@Iq%L`S z>UtoYJ)|v}_;Ew->_O?1P2%6#JBjH4OXm z3?XRX+x4qH*?Ayrq{crsbI$lV0rWGwT>3`z}(5D&$OrouafR>bCPnUaB zS`J8tyHl~i+1&!pb%>W{jdJ+y9!GzOK6B#Mo8wjLU>$5KX3wSODlBD;TLB?7L!Z}G zo<|_H&CnER2cewy7N<*JT^*x&xh#mG5G8%vt1@e^D5pol7WAMJs9HuyVy`F1&!%5?gBc! zIE*zUl5AxcxTW)AI?WfTS33b^g>jH_?`2J)UEtmv8qT>OmMe#DUuWR1>}W%vJOLmF zvvsB2yfHD3>CNd0D`N*@jRYcWm8jo0&@rm3;(u(mNoU2p5UoBHld|r3f#^X%SI!fiEXpen%f#_* zH`ojK&Kf^GPXw!Mv#W?`nUK8%LB`q`n{>t=7!rwY)wVp+~8U=Y!Z!K5HXrFRUB;gcJkdg^epRXpPa(b}^t=zmzp zZ++HV_iQw#@d z`IO6V;0rF`zO1ib?7rEJ_s`zIl-q^GD{8{nYit)GBXnAN+X_HI!JqHEoN8>)v6#_C znQh*s>q(zE7Eoyh+g`Z>kI>bM?j}>r`8Nje)kCGy#({upNOyZ3hepCekAFi!+>P95 zJ-S-jSOZy;Obe#j=p>tEiwVZf?RpY)yh?MsBs``E|wDa|wPLY_8uFs28T}y%Acu z$z`1Za*C5zw!w*!nSav!@qcEDIR~9_iA%8yx3*`I@=@z zgJu_WV%q$zw zmWkl$?ALb`E=F_X&jNdl`+gN$Loj{}8xPNV(j3o(*fUPmpC2>R?te#hsKT9&NpXCo z7h{7Zs*I8lY=C~PL4~+=75OWuY3aKPGWbtm_`g}pczh$h8)P(;G0|&xc`>%_U-l4E z$+5>)q%7O_{jgCO2-K?IF%|uV$FDdzj!}BwEGGUZ%(q~ZugcT)JbTx7KE`SS$c{s&@341Q+{b0)V_R3%6S7LBQ;s3E z;y7qY)>#VejE6Bbv>2)}BTW#0WVXt)>9ybCAZQ4WyJy6ksSi&J$nI;kg02@#8S@cb zwxcbS3X-3&P+ZXO?;5PFi$AU{t1MZ2-yD*$H*rx*BznknBY$f|%PX}Hjb@d4eNMpH;t3u z?s6dcPPO?lGD@{iXBh1^gw*pNW}XuE1ugtz4@QcqkX#n|Rm|@e|BmW02HURC9p*-6 zyVl$wZFW7KvVYgP%L<$0@Jy5;*(m`|v5Cml5sSC&FuP1hu~%b_;UHwsGt)I zef@Or3%iqI4W|wtiyJk5E=99NAk4n7!}|2jr-1{ zJm|I+OdrGe<_o=L!<$jw&p5ObJbk) z(yMdX^qqR`>rd2=Sf~}Rg89s^0iY1evO+KF+FxCdW8gJ2ohINzGu(@1t#CKaZ8%~; zsDBzW&zR2ygsO4q1!E1nD3&T&uDq@;1UwQ)V-mJ{G8ZJl3=siA5AM z8Fcw%Yx%?xt;WvpTUoFD4J_Kbg)qZ)vCYQceq;pgv%jRFzspd0<}cZVCiA;3U4J9= zOdX~-RuOtJW7Cd=jxlERj)7^K0(L*udBx8O*$^ylVqLAzqqs&l0gOf3g!mWW(hMBO zz7!U{R;vtP6txb9=1Mo)@q7`zJ|xG;IzmD^MjB$@EYB_hzz`Naz=I2AcK)RUn@iwb zs}VKT4o4x#MB@`+H=E^P`Vj0oet-P>@*(tFlvxZCp097DXXT%9+ee*}*&==reHtwWqWprnONJIW&+j2tRb-es%<+|RGF!Q;P+Nwo z`CGZEFyBs3R%JGw2vMvxOMg2{0p&NR{cLlYRyH&Yuf3Y+eikC>4FBnAaP%o)=ocA$ z6^C@UBd5cbZa;AKeq7~g2{Mz4+>Qqg8&{Xaknd1Y3LC--CV4x2oU5!O7`bIBI5s1% zI$@b`ZUNq2JYcOMTyg2H32CEGidnI~PEWIWF}sfU9_%`E?Sq9G4SzNxFqzMb=Ig?) z{H|W$sjEB;W9D;sm*Pgz))CeZ69R~_B169xe~Z!oUh>Lb7#hqRey0|XYqq^Ot!qt? z&&#Xja5*@};Z*m`+>NAl4f;D=);hLioxpL%6X||?tjr_9o8wN`%j(4IdIQwV_RVXn z`AN5uBq%E*M@J|`N`LOrvstd&H-^$}Us^dh{;354#|PrT2P*`G6p5}y z-#kj#gk-QQARy1Z3vrW@(e$|Y^exxfzP1&-%hrgSQehc8B6nEdZM~VcR_R#()dZC; zM#x?dEhyy1fQARLyK}ep(SDt)h%%{>$!5t2H|?~oQ?=ELAb;rxHn#iZUZm~Wq)AC& z7@1a#&{-n_%|FQ}87OZ;(JQYOm6wXD*7V@E1Z_NMwHZ;CC^U>#G89N930`N#On2)v zHzM{7eH4>rA(9-zF;*pNo!~)_(R^xolDupr%7Q(R@X)S$p&7sITvfY6XB7+>*FwE9 z)Sp|?aC{96a(|7=W8BoGn#@@J(XiCnT|1vGLk=XaEib&*??vB}djrhFW>IY*m6_1> z3%IF2r(|T1gF>uon07QDhzZO*1(1jpi*>1dBH_y5kCnu1!YtSF2#sRk(*(U3f$0Qp z`pQ#=Tqe2z27qZ4L?1^Jv7jIkVIG;NOTyZw9c0kOBlgroONw5v=| z#X@vZ=K&#n1PIlg0ZIBqVnyQdgI*B6Gw)Zf(;09gw7;X|)o@T1ZoDX0Y|4VATI?bf`^Rt1<} zp+qq&7dOLRR`YCl)QsIEHr7&7cw+Ipp|sI7o;~k4fybX?TD7;C*#Ak>=}%?L7V=C4 z{z$(YhW8~uhs;&h{AAeMdY-z>QHS}t@!Z^IUab4!T3TJ8)IbO9Y$ioiW!?W053TH^ zynjTr2k_v5lCKZV{8#`Hkw7fsIS3)8ajPM=oxJWi$tD+c=Ydx`p$m?t3kG`?;d(J~ zzv6d;qy1V%(by`6qv*R9g~(QhoK@DS{B(55WxHTs(Lh|eF}PqB!zDb@hccXlpTf~T zdiwO0I;P`K)kVm_J+hPD-R!(~!(RK2^M7}nzte0J?6BFIIczkcI)MW2whBr9?EbR% zRnWMWo<9t1qQmTq4V$FMpt< zOSBc$wR2e+A^e?^Qb`{W%XBE^aS$`16s4T&HHr-ZpW2a|fZ00rMVnwoxNS7(G17Vb zTDG&%Ds38_+X_0@q~p-#s>4_n`BlEmv-K9nZj_wo9J3 zk~i7Kn=Ikp^4vljl^y0Z$4;!qtZ&@Hg)5q47dq;H!BPA-Wk`26<3F>u!+-EQbCB6L z?fOiR2ej}3mKTI?jUCx^9FR`OVKG&DUJ3>?=B3tD@}IFke7~I*H$5%VYB_`0Mkn#< zx&YL0_v!7=79Wvon$POszM+fO%|-hWxY#GdV0@n!pEu|j8^p0@m@IC+Q||bxFp&s4nSU@j3N^*#Br5y! zDm5ZJK$*f+lz}9#ZU_9gre_kS%4Uk-{Em%P&~yPEin{n&`fAS3{rfZ_AGyz53?JAb zMs?I3XzFrt(4}%DG1CY8a9sn9ma~*}STWueNyCWSYApeQ=?pxAK|%(zVkHh8h%sXP zhB0uc-ei2Pi%`sh!GB6ZQDn`oNesM*(VlGs5uCo*nHbCy0?r{ACyF>Vjv2@O%M3CW z%Se_?R(ROP{RA1x(cUp@e_-R^KUz?pZt7V76&!9eb;PeBK+6MEr9{k_KSQCdO|9QL zR@+F9!Mdt@wkZ3{dJ(jt-cV&0eD59__A+vXsb+A57GTJOpnq2iM-XihYiolc4quX^ z3HlS1%74v_ZYcALz(O$FHd)?~Xq-iui!g%V*MT(kL6n9Yq(M!opD@e-8g`f^+<1r^Ha4Y`%g#q`zKSxuap zZos{)l)BimN{@R%hh&N&(Sb+?%^}NyZB{L0K_xI+HT&9%U<$9+xjbm>TkZDXEMwX4lMhG^!!K!405-xS+r!#B3H)*9EnZ`jst zWP~=hyiQ~N+gPxh z<2g>{6Sle5FQacH!_XCUF|A~`^l#h1)_`SYT6#>mDAuHmkAm?-@T)upXjt>QHe|n) z2sPXWyMIgP>JDDj9o(r~alfRCEUiW9bgUg%*ge2`s`Xsvu3W1uMIKtoD#Gs2D_k3W zMHY06<80W9JKL5Az7xicn%f9@M_}Ht$y#p7M0^O&qIj>ndwyV5bCcif#&5+1%RC$V zTuv~LGgC0QjGxa6c( z<*YmRn@P3=@S(@M^hkd)7Xt_eei=}2YjQ*Kavr;F9qf1U+Bc60b+p9cP?TX8 zI?X{UwejRrFrIERpn>y{2BlqA@$4eo_kV^DN9n$9n|fin*SQ4-NBhoJI&K{rz1}=x zJy*u*tk4T~QY>@G`>X*-iVA_jR{1~C_euxc)NCxHz!ry)#oK5U5I{w$svcWIh83NZ z0DQYD@+#q^7Uqog7r=Xmaqqk9pyti{dQE%?*`{vAwTvmZxdJoay~bmf3sp0k=zlZ7 z4lx#9rCz(F3j!XdE;J1VLzR(iUnj9YBEbie#YMKLLx~;pHM}$A9`TXM1zby8`OSf6 zI}0nu5;EB$rOPw_E~fSlj{8oUu1G$RzjhtuC$4<`w~LgGxPJTOoToB?;yU+JfVyAt zvWUW$j4Yx|!`LcJvQe&`-#$Ko)qk#lPAF_C`nH%C3W_D4j-EY!C1rp2({2J)YHYZv zT=|R8CwKXehtcQDLblv>^2s>s0;2j7QCzrnw|5-6AQ)E zW7fD=P_U32a2TB~|Ew}{qT7UgROjQ7kDGRtk&ty5N%Z+*8ros9fVXjJwSNLmKHBTq zVlnIS&)kDtEGfGy%_6jHY-arBE9eAcvI3WV&Z;0cx-9Nz_S5!9Be8)wP0iU2n(W=7 zK{O;*`EFWFKtP9oNMMAOtBF7E2Pr#2(Rn3#}JhYap7;@8cCVOI&kP2 zhY)lA=wNOLvCefC)oB5_7fci_HI|U_t0k?p3(T_2C;A=&+F3sU{UwFfZP756$S|sN zJDY2>CfR6+QDck|KYtg+5-ZApdWA@a=4!yU#Vyh_sgtrw@p^Hprxk3OW|2x9e@+oD zIvQQK#IUc#cg4JH;La_2RL+EWp3M%Pi}qYY5bA70QN&d}F~x!*x3-o{61|450w+j1 zuR6h$#Maco5`Vk@?foCcvME>V5gef9Y=cJ{vZG>wfjZPA41c_Eh$)#ztXaKUBbgmK zn@sX0usK0NX2qi84iesG(Yly>)JnL{on)xu<}&VIW(SGh9F8z?6t*mJXAcvI3szDA zN3`;olT;u>YWf2m1xcPV<`p0#T~Df>Opp2G$kW9PMst8!m-1(e0|wE zbCPU2js49Mo_}-E!E`N_EjDTXPRK6l$?4t~#BwoNsDBzm$3x8sdCzjJoQL6*y&pDr zbl3)96k6hs^Peb{s91RUSvI+jPQ=o=7(G4wvjug1A=YQaVqIH91_So*HCEnneB+@! z%s9mhSt-gh1mon6oh=FoNdd(PPO(ZFxF<+C$`uY}9Dm+2)zDFW)xhPo-`hXtOOw-^ zU4_Zd<2b-MYxBOh-`LP$~wR4q1}`%M%S6 zqRpn|>0VZY-^4k@0pNgi7ZxnAKMz|r5n{eMxyeC*Stf@228U`iJS2SvjvxC=D{PAV* zZGZN?xdqq)wKb{fwdx0RB-uA+ERzY$+qTC<<6Jx2V0#{L+f~@wy6&s-rYq84mBE(M zat*$2kw2S-S8r*JTNW8=|8j+mTUQ(}2=RL=ysvJ_6Im}vcW8W$87$twNt^2^P{G;2T-u7nAxUXLwUKCO4n=U++B8{bv2D4CMe^fvi;uuAls|0?1Bm)o#r{wwHc8t# z&7cGS;ZH^7yJb^oGgHVrP^*|10{OIDJ<2v!Hhb}s`ffx+ev?(t7mIxLntbYB7Ju)+ z8}GZ*C-QC}dR~2VeX@d!mDhxuSz!R>XFWNIOHzNNm#T3o@a)CQ&tEb#ux%#a!KStZwv$;@(?MSqawBr!@TIAkkK6bHdW;s_P_s3D_P@PHX1+7LYI z#8=j(>^gzSQtwMlEb6vN7m=n@DO^wi^xL7ky)H3HDlUfjCjF2xQA3&8ER%QZ%%ZJ!Z_9}U@s{$R@(HlyoqtV_+V&VR)ERiN~W z-&%JPeWMKAKqtG-VwWkMC7f;vo8<*QI#f^MUWaj}8|1O^H7>y9WB9b=YHM;iy|ZWm z<8xbi^8Z?$pZ%<}zhWO~9s$^AS$Z^wrVk@GG()?;wM-C(&S9LN{)&e`cRptGlT4nE zXndvHL)toZo5j%?NO0V*kAJhihvrVhL`L1n_;KuQl9$j9JVzfxfk)X4@dL_L0;yJw z_JxDvtYb=kGn6qnp?=dNEbB6YP9ugE?(m3(N6H&0{C~*y42$4+7}*c#sWEEXB`(&h z^EAB)*u;9Nd=1?ALm5G+A6p;|;A(=`!%Y#53{Zrsryi4=p?mky5r5A@9bJYi#vr`* zUc-E{YY%7I?uHx|2TVncULv(iolmr-9G|%pt};^saHRAZUOlsfS6vO|Sp6})(K5%- zw$sgG!Y~9bcSAj(k3`LZlq{xlbZBa>C2_Z13Mrx_$2 zsmkc`#fz5_WPi0-V}69JlH}ya;!LdhkysUHi;BK{zfR1Eo(&POU_ep7R>tP{F;CWVwl^|w@m;qx3d9&J(6$JjE1y-fh zkxr$zGP0mcl#r2x%5k3uYbOK^JHK0!VxLBky@(-I9r|P zs{$^3@%xQiL{g03zNN{zEGo!K^7bw8mbY*9&tarW%(BJVCOgZi-&C@vyiA>!stDAU zhkz#S6s=!N1^pgi@lMW_Q5jrdzkBq3bRYkE_@j0`%Vb*7#U(A|Bzk@-LzG##%*{cJKRSHQf%NAfm0i!XTBa<{kmh!r`j&M=Jzmz_~2X>XI!oZG1vzkKKX zHr=puwO}SX_4Blyl?87uNjO`fO|ZFJ3y<^ht$$2CcWIz*5ixN5I2-m&6w&(J@1Vze zg#+G=lFX1L*6+}?X)|rVFNt*FeR-2R~#~ zU#>QQoh~Ya<}mfNcK=I_^?8s-45X0A_t^@N@vd@-Konc1`}h4~E><^$99@b-qwIee zFn`u(I)IJafgcO-OnS4*78M{tau@sBbn$N80$v8>0jele3T}3@DPTkq(}H95$LFt~ zfA>o4jgygfxG%}WdR;C@v;30rg%Yv>12Xlt1}vf7D7KNbo);&ey{6H;fTVX-z=M58 zNjFMJ1YvYHiudx-=K~O91zT9pu1TtsqJIQKi-sY>xe^jWd9^?iUkoC%xk(u2mmAKO zOr4}Im|u>n^FoX?I1wvF7cchZrd6)UjC-ARM`9a>JKc@YD)X;Kf~(r*&DH_QU7=W| z5tc#lO01-`&0HoMz6Bni=BDE=8=(2F$ZD`VF34e5LNs0H6Sy|6#z%eTaPA*B0|IXMM*kFS;8i!4(`5vv=d@=mI^FF2Ey*DFD9m(eKCB*d4nP zq&#waJ+&g*!m4_8b6i$!`G%wP)I&iI0?@(>W9ZG-;ksqUFCf31J<$R^;HBcU`J|TNR>QN*`XrWWpFk&7tteX5Cq?gTy z(dU2qfi$wzZBeBdxl#8#15hSLiD#J+r zDG13}=6m%z0A3hR@?sW?*Y~keLJGqN#X|lbd}$20m^mm4BO}KQuM!l!WL?J8EX+&s z*#r7wm0}7vQ7|E3{uIrivpj#h%;S0}RESH5LK2Wsj{cSZ-RKkf)j|F41YhobF)%lk zAaS&>Cw!l#=r}(OJY4EP>1S5YE$3}!%9b`-d z1!5LcK@UT@7C8l2%Xna;M-V8IVtlH2y}?-;d(H_ywvH1G^y>DB5(T)W!#k0SR~>ps_bZL$W6EQ+i^38$l}6 z!@kOBWq-CbYLpriq~w3Ad{B(IDOj-fv0j^ub|Y`}ydXa&u+NHdL&hvw@=00FI9BY% zJ=Qk{)?y_VTrV#q^>+wBLR&0|v-8}jg0f>)2N0_@s~u;Da7QPGa^L%0gLxXcFVGL3}Q1&NOSbDhpr*fO1Q6o-vwT(F%O zyzb8_oVTY7MNoOQDeqv-n{@r^ zaAKYI-lsYRt-BN5r@aR zS6e}7k3g2ubKE3RULc#T_pI0gFaJ>=5iVGRm^^QM?7#z{V0Aqd;pI=RU>2)ufFRyL zKGL;xDv!-!8%FOx49u6f^u@E9z>!>ked%g1R~b6K^l5*5;WAGy>v zQ9(@t90sasJ@@*8H;RrlJdDhhWVi5ms2<36DK33v_W3wjmP_g=!dW{qGb1A&?Hxmc zjaoQTPtCSLJU-xU`0_m4_@VJhCC$O!tXyU2Ck)4i_50Jv zg=ZP)1x9}y%fR0T)|fry%){6@ z_dfEW)|ahP*Qfno?-2(WX7T``4ZJRK563Mh-Js#9W4d1-eC-^um33{;tA6*}f4}zs z;KG0BJ_X+1R~Sh57njAf$nMYA%Vc?d`)}-u|Gs$m5dRhbIsbnC;K9Q^^^BhHJ^buX zpZ_k}{W%6;1IAzR`TwEwe=mA{`1l{A7eW97C(7qjFg=_Wkm}-K34Vy92g&YEPZoPx z%+OVO1^nQRSStIG{z*yM;aVB|zNA+o`b&RG^=qdmeR0GISuD+Yz8kO`y%+UIbmH`rkY652+-tn&OK73@uY z^)-UyLA>P_^)DZ&>W}54A0N1J&h6x^-b;7+m*~+rjLf@plak6(s_VJwOU=RxxLcKF z!qi}EF+Ov0E?Qi(<*cxYaqL$w%O`*GNx%r}z2%e}S*XM9s<9rDB_=LPrHJHu1zKs1 zU^I5zY`<_V{|2In4(Z0Sl-Uh+$opV_C*&O0#WSw<(c5VV-`;T@(3(S-OmDQEX16&e zh-$jZPL=bu>rc1S=-allZD2PHxJ59(MPfaN)cM_!%FhW0Z45{UFEDtDAs2sr5zAfsVT3s zQN!qE$5g*}-BmWp@6Xg_O0R-Ppo+w=4@54(zfs*}KyT8)K(o zW$f@tg^f(TI}M9xr!8YNo1=H9Qv&I5f?m5mcqWT+!Um7Qnqlvq9S`)t9%Q3*0QO+G z(+YTClvs&PTyCngDWG)c2KWX>t3jB8`qA{`WO`VGsGx(SUvqk*@&kXKGhcn1a0SG=? zHUn>2Na+T5pR=;O*ev&>dRYNHLM$Z1!LgPAO?u)j`saTMR(qaR=Q+Awl?$2dp4c&_ zym!XW-UZN}*jVAoWj5PDa%Xqgsn@ev+Wjow*&0X7np9n z)@BDQFil+)OO!HY&*1kyUZPpcj0-1;$8oCCc7D)o~@&|ZxxldZ`C!3&=-X;T~gRQsT<%6v15M&9f@ivQOfGTZS$FV=KjaeE~A#Q z^UT|4^z1XQJJC4W*Yj|!>|{;}H95!DhF@gM>Ri0|smjO+UQ7s`p_LQ<{R>Z~vVZe) zDc|x{RVR(QvOIiNjeu!Kc19w&LNsMFZxkba$zrj^Y=v|!_Gza*169xoI{-+gR;1 zkLWp-&0d>CA2F)-1GAgA#PQr=fjmt?)4=ci4>u}JJ(E(?c!A*NT85bQX_pzdjIY)c zq={TL4D%Hm4UPW5sXP2+8U>DAy2tTO0J48GVEypb6=s5l4MX56L{5W-K3@xMi;%g~ z!MzJE&~9t$#_Qv@Q>dTmBmJ9ZnVo$ z8q(&qtQr;7;7g3yy2g4hUPgE=Q!ZxLLk!S_Fgja{%!ST1)uu##se0Ify33X^i$i}B zG*<3zqpqBli8!dQPRs7~Qf1YL)uWUFIzX{SKoIu_F;hO1l*#i(3O z0gpn3Sc(>0!9kD1bcL98u&wGh(b=p#$!4C~JU|NOct_Hn1dZ4%Cg)%on|9m{NpM|T ztEFwLIBe51;l4SHjWh{i&mkp)-V`yBYI`tgqN3$wWOTD!6=zU-kzYyM*w}xuf%aK| zy8ga;(Ra?5K8y6mmzSnOi_?5GGj_mxWeXfBE(1Yp+JcrlY&Vhfy2#Tju~}*tJSUIx zDl4iy`VpAJ6F~8a`$#Y%W2E3H$aHiF21LV&0|rPMNFx0~OSS~&(&3}#PLM7L2GXj@ znRk!99q9P`WK{~O&g--SSWth)0;Zgk0is)xpMkS7bnLe2S0?*XU;m{o<>G;t#M^0Q zV_F2ij}@X~=2qTwbKkVJM=Qn#xNnjq$NQFYp!$e~V#bi}Wwxv@3>xkwteo-gbH#>!`w*l#@I;M7xFSXIB3-g$C}~nCxUF^ZXZQCeecp`?J+c@gSgBi?~tL# z?xm^u)uUtP?R>W6OGqRYbX#|CbbL#gwb%moF!3q4bFwX9iFn$0`7&D$O&15wu4^~d zBN*s!0+P&P3SSsBRPcWat6Ztx)g;_RU&zko&3VvdWM+AB97mk=E>>fHg=QQDU)Hqpo%T5$w!ZgkG)~-$o}pSg zBDEBeBM{B!3IJOGg%wr)l$7TbHF}gp3kX(1fI^GfC1-K~Tu^`1!AJO7YVt|busXxj zd00gYuK@hd47or6J9%%apAzCtq%rF#NyOS58-s))oZ_kFsE?LMhO1kmv!D0$oP8?& z^StMVCSkrak5?+;Hq>s~aYN@60Dgd^|N7Jhr^ADpA2Hw8!KsI{z?_Io8^ObMr!BcW zG{ql}{eUl#(@lT=3usre%>I~V)or2k4KDd`yX0I`1&O%vbuHEOgPKD=lOPhhmmON! z(k5jX@qGn>in$~2GIxuyHrHfsvO^dBtr_>t4E4}({H*}OT!&h}c>U*|UzouQ>;Gin zk9DizqmsE_vxHbHtd^Q&7T;()u@BS`Mcy zs|R<-OXVa*-m$H5tRUIKRK4V`ejLOQz%KxoNz!P8Q74+&uk^|`Uh|rQX>lP*k#xXOlLb;qJmaO&xjGu366V34sC;#%{4b$KsnrhfXJ*#OIR424F;=* zpjv-4z2o+(NqbuGK_e9(;zfUc5ZwAbUcw}wrB)nlE0wSRAy^VMN!}GC%!lL@lc_$a z$mrj)0p`gk+t7l#wZraP#zt$WX!qZ9MSh}yRO0BlB zhOiEo)r)joFV}EsvOlGA0Nhac4nu#3FrHX>zqyxfoUnlzwD4KNd$iX5%{R~;^5gce z$#Dr{Ud6@+q)!cQ#AZk_3<}nrho)ge7!v&NE%zQK;51AH2DvZQw7$s^18efeo!R0< zRKHF-?bx;5ufDTXN^Hd;NMC#ON$1B8wQg*x^FW7&!sJ@sp zMU^#oR7mG$%mp7o0X2@ZW;L`SYQ`Eb9MZZf7t+_{HPyUVhT9ZY{!&yR7|-ozV(Mm+ zxoeZk4NWPYV`Oj>o=TTVWgv`J%?w(@)=az0{!ok!=K15dpQuGvR4^ z>#v?jdo)6}E^j1wj&a(D0Wg1J$OyFvgjQPY=CCNUYd=J5USMK+f=M#_*a#S)vT2#q zyYm}{@Xvt%ix`ymPxcYVd#W1pc?z@AnTT z6l+oGo3wIC_dwCxx3K=+zQurt>@3Gz0Se5Q{+Sgg-UGx>?T94&O(uWN9Sc*KDngP> zjMB(8psIc}jKpBfz^h6q5c(D#H~6iIQg)_)r?1&AtGhire~Cmug}_4<~6jN}5Nnpws0H=eWHu>thmDE+)R0MqMlKJ_>cyTX! z^WEd`_M`9fx!6Ipjd2s??*I>p!uYUlBca><>tC}xIyirP9xXQ~vtpvsW7S_saS_5n zvZ#tl73dCLzReJSmoGtnziC6Ud70N}?tDkj#U@*cQ*99#4F#;7oJ|)gfqaI`Tud_+ zXdNyDsu*DLorohaTxvPX8(-GpV$H<>Zc3II==6x7;X%>ZPyh2h?PQf>5+)XBS{~TI z)6m%;b2LvYt}VB}W=x9Pp-z4yR;MUBoCI83Z8qT?XT~mxyU@2=7}#$9SGa9^%kQzA z!22?{$#3^zmTh?7wOGdOK4^0vzlZ0O8`IaU%cIKj^&6K#3jrB_{3Z7w@E3z|2;6d+ zo5zXw9q%Z)-N*cs$@`){mTvqd`*8CS8 zd(wMft55hFe~6P7yzdxN!tFm~iX8s-F+`2qf65YE{Pi}wgJA=yhof9eJ9qL8LwI7p z213)_drQ>A}t#*V4zU$eJ~!#h{hNY@)3%Z@>PQ~o2VWbJ%XQ76~2mgbs9i; zTF=A4POd7y2$wxhIm_%o@$}*tSPM~<>I{6jV6p-)Vu?Fg#miZ?*atfj&LQBv9$6tB zu9!XnIeayLDz6rp-x!HW=*dbZkOz|R7QPpiOX-JQC+S?~lREq#3MZ;j_$nR@dQQwv z<>~!L$CUQ4&PQ~)C*-`<1>}Dc{|JetPE8bh;6A!YI?op3kXh9l4J2@M-y*<=OzNyO z8zVn%WBJ%OS_E4W4=|FhD(0KnI$PxB#>u1mNsQNjeUvDd;^QbYs9b*Hw+ig4+-Q6? zaz^5nI`4={l@fx~w1+AM2tC5TneQQ8W$jBV1co3V!&!D+Zq~8v_C8rHrNhr$J#iF6 zNbxDYttx5L7CtoU4jMAD+kxVe($K+Rb*sl5(isg6xceEOriVS(i$GUdgWXRsb+?v$ z9;~PXQ@xs((hUJAe}iGLw9)E@8UxSO6^p_OBx}0v48QbYb*UoGSSda!&d24o>Z1(v z?y_(#lj#_v+9ShhpT}|U)U)-{t`dt^^O+tFn_a}=x;0IWiL1>EUBJ&|NW1<|NWZQ7P7);LMop=&V~H+N@xSMx{vq1@_gvxMT#^Ou6x!plssddbXwKR}asgRwN4Wsf} zdU2T|*fZf>pBjq9OZd!dA?%7{OyVy-e&r5k>^_iZz+V9lqnL<>pO;e$SOh%YHtJrm%nPeXgOj{66;F`Mi)(#Ism@*^%Q7 z%9O0$e{epi$MBHm;*s|0QeW1m`gaRRbS00y5h9BiE?{s&1eff&m^XFw%lVD8b%dJ# z_N{?pSDv0$`5H1R0JNt#|H(;BvYAOKzE&+ujQ>((@#Z3tq$HSl3c=t&L~OLTZ_On* zDQBDc0^SwG0ttE-*H=WD8Nd!AkM>DkNQZ{He~m*B#LFwhNR@*^(h{^%bWyZWbouZbnd@zgwk!9kY1h*4Bop8svgOgimaSyGVQXfS^(M0$LGYAIbB58a;h+Aehi*e{oSWx;VQmi>Y7%hd+$IdUW_h1X2&izzCiO zxs9;An(Tq(W5%P_0ie<>!(&R{iylGwk+e<|Kw6FSIh*w~oTXqsTWnXWUIUYDsC5IT zat~8Do|fu>0Zkr6?VO;l(V22ptV?v|n&AOLz_z7Oj7@P&EejcyMDQwpQ*rjwf8~1B z+9`QiVN6bIHQA%@$+=!aqQOZ?UVnN`p_CM5lT|6s7ASm~&ERB2;f_vbNlM~qUY$b* zHWE3E(r9k99rK?6+pubHxSGZK6y9$XjF>SKGT3HOKw3|eV^MZ6 z6|Tw(u8x=&hh7|Um5!x&tEFc6fA+p1DwB~zUx-wsT8IYu`bP4@tLIO@`~GG6^}(A* z-}EHNHoOsCBLqo9g3vu9!BS5D27unx{REbaq#=oN9M~!b#nVk8r>PWO5ZEU%w2Ew2 zoGrlHVyj(Ab#WfHV3j>7L?M(tW%f?j=OE&hv#DBIdp#e?+{Pm`0fg8|e;GR2o(XV` zu73bVseT_qM%M+%$gDa=zvLaszKsNGkg`$DUD$MRB_`Kk76aWc;GkF=DT`PFegT_+ zHEN=+^}DvaYMqfVO!Iewsu%?on1m|;uH}=Tt^_XwvlqE$$h2kd-4yPV6*)CyX~hyw zOeNirqjYbppm^xPxOJv zw7}YvVt;{4!%8Pg5Vdi<#Gx8}A~SL80sPbip&Br#XkOod9)B%wMO^x=4D<`4r;A0* z<3b1O3_~FFTeQ}@Q1N(Q7Y%4^-TpE;&qF&we;Cb$O@8Cn1~tE%wv^wk!;iX7{+&C~!TxgNUV!oCqYnJPTf19L z7r!cYqKu0+8?4IBYLd6AAT3lbi~MQ}8gxT%T0EuoV{AmS!BxfU!zYi@$4?%8_xMTr z^2wWn#|Li?(x(Tn-y9yi`Nkn?>8r>s_4tbxRB`9icrKGWXx#!v>Z5F7ZUI{RA+>wH zAHEhfQIoq^2J9Mg&=LM$)(V=6hjlsu_CAi8)8H=mJkL0NBwWZ*OZbA8j*t4b7KXu= zLXSW(LD)Ssf6MfFN)3japw{SA<}5djb(#5WEgzW8#(KNaU!r;9-7ZCc88>cAj}Ave zj4M4}wJTkU4m#CIRxz8!P2CzhYVe?ynlvgEr>F5eaYPLuIq+y=BYnD}dQmB>1v+oo z{#@pF7uRm-!N5~{>u5ustw`*A%Pj0?_Ne&>axr5Zg0F8A6IH{xRVDJDJ=#46+%7dxA*i6XwvV}HH@8_;IoekrA4dEBYC7igGY6;| zs{=0@IFr0{Y&0Dg?Z-sTLJ+?7b%NKC#+|5i4=$IEXYkV(2kE14et7k5`uh3*^TfYhWUnj7^#!NBwYpu~ zLL-1K&4r}I-`UF1MRz1MX{tu>vc{0P0y7g<0voF}>W}MkZ76#fblO1XXtZ~n6zhB* zI|@~u@Rsc~J3QP+{Od5Yf&N>2Aq2R{ zz|{GsBY7Tj=|vAoy~RtRq(tLaUPbLgxFCNwQzV#j_CZ`D3IZa+)=jPn7He}~w^ z;Z_iH$AOb#g1V*5(ykNsdhap6iQ3C|XuwoiZEtMH^az}0l?-;Uo!f?8bz#8Y=6dZN zEp29BB*rS~!=`SA#6G#_wlO7IHi@vsfp4U--QCj%tOU-pP+56j!W!tzNtelSYA?`X zz|B)lxbgp-rXV-0PQlcmZQwj54x!kfx&$xs*7g$K2Wxwm?-&6he^(~az*hs5lWHTR z4k>)NSk*pv_6fV6miEYo<{kx_avW|8{O5g%OS9KB+FQ z8Y5g54khUyTt)Luf3=R**#$*vDKQgMdRC<$+cT~KDQaTW#gkIh0T23ewn4vokU4df z8@Ze(A?xGNGGq>Y>*J8IW_pfz&9RaE5||I!$LizCZE?KEms_v$OmOORfMU(xtsxXN z6@#X1qQ+4k27efIFqnGV%rUaDjM2Bnam9jG&PJMn3jVQlf3!c0c8^;PwT&P1%Z85g zrfQO}G@a}LSp2|Yh#|gEPYK6G|5xBup!MM~ z;(I}+W$1~+6n+pIDyWeX1w!kZ@No1@>X%K^=CwvWIBMwdafrbHuLyo%9PJIlC*!^7 z$-BvHBZL^>f2|`??Tiv0L!gCff{ag93=|Qod~OE>;JYoY*==Zz$tOW(;i)9xT^sKX zWYC`)gpt*X=KCH?AgWp}mRMFCY@5+HtAVb=`WDR5|9bmJoi4Pu#90>R$SN-gOEiMh zdA@h$C_{KGTse!^ALhe6pTh4*{lPx|yJmf)U^lr7aG~&1_vI zUK(X%biRsq`4O_9;n*eqX?Ph1mZDXIn@FEE>^8hN4`8m3^=2F!y)=dQfXqs#vdmM? zy5F<%e>U&S%hqkSgSLq!yp$1kqziN|AYV0mlsaZ5UJmcl0bT|Qdc~^$cALEZI%@^) zlY7zAVz$m#(aANdC5SUJmq(3zyad#q&O+FB>JQ^N;t@Cwk9AjJQktvXlN8uyHoMBM z#h(PH2{$2_SRQ_y7OQH_(BXphqZ|Nt#NXwqe`giSW3HC__qgp4ZIqWg2*2AX)wE@7Pv%v=1ce9DWM+AGnRiyKn^@G0 zy&^$7$7a3UtdWt7Z_m!?zM*DfX8 zc4E+a7-(BDcYVW*-8<&S5Om3{PVXDcLERrm8TvP#m?q8lMw1g7;M-1+6?W2Tk{GS^ z&L--1quy$BxYLnWrj}uomHCGEYrJKPe{)j}npf(J)~%3fwnwMA4ff2<@n>$1K@-vn zJq`C>^c+B2=fyiA^lF7)!a$jpgJqu2L=iDy3d2oX(;b3R$B(J@2nA%YM}gUW1_0M6 zI}=un14={~kmtE~KX8&&HSXM~1Sgcq29^YDi7%rPFzK%#Zz`g(LZr?i?OsKFQ*ikcz@~$n110HSN7>mb&l5JkFn__~>EEn@L2aGT=@QyM z9qKk(YPtHXjMGFQEF2_|Ne z(l=q@ZqGYDf`C4_^P?vyT(C2FQ=Zg`dc$odSHF%SHos)R){a{u_T0_Lf4iJp-uQ`W zRWUGKDZOBr$gMo!2d=#K6dYK;eS8p2GKfW3=POEw0z!_wBgxj0bj%nLs@@6YdCgYW z4k@^75z;?-DPR^K5P!s~Jj>T3^x=-Al$6+xzX0Us&;sw;Vp1%{_lJ?RR-zz zk-J$E-U~(fi@{z>F*$IGf4vAhZ)?*0y*H|K81FjJZp2%#L7oMgpuFOfNR=)lJa>{> zbu^crz2C#E#tZKNPqoT9)_=_uNBh>rFK-X$`` z==*JBTrd4Xwba_p-_fV=%Njq;jks-J%(|U=@x{ZB)C+%E++toNgt=#`4BP5h-{VU&**?0Ne<>^xx(Oz^>z*D583o^R zt8?S+dTmbfn45977Lz(*9Gsy>OSm|NHVaC6iE##aD^EvGUSQr#NH8r970cNG`x2oedzbtVK}6qu1JVx zBZdB~d@n)tf6*8^35%9fR0b`4Hj*wVQ2D7?+Y#2L{2SLp93#*;#Db*-bRXHxeRY#y zkgL(k^I3`TDwGv_3VtxMt@SXIF+sUS)#x5A;qG0KnLKx`1|Xw`0@fsN3sn^GVC+hU z53&lFCBlfpVKeBy+rwjtX42hrfI&8Cu&OlO^tZLOe_kuc-DrC0L-vmoWWuqNU@OuWC+9wEpQ~pzRaU2|e zrp|U_&HL#DTj@2RvDv)_Hxv%rRt&2lzYdpeb3;VL`C0; zv#lHLe;s@A%6O7>if(D@wQ=ToPJqF?JI~iq?@_i_98U`(RCVHMe5CL;n~L>E2)8W6 zz_`Le2DUf&`9z84I8ImK*jkiO!L~t+nL#50UGshDzR|RHzIuB(pv~*ALz8{3Td$a2 zShw8bUbw5U5WIBXcP!I;?L&Jj2B)boOfJ~Se~yf2nfCbawYcTHQtibwLjM=ArMSlQ-Z@Qowa*UzVE($sxc7{F-vit5#b=-Vzv?C_A ze}xwW5fTIL;LV7Lj_lkDc*toRtuTk2G{rkyjBD2m``S&pVT@~s3+u+vyi3KFXzD*FYPloZ(u$a(E7tNl zR=pi?g_2f@vk~Og6WY9aYY@%0!5a1nvse69sjgn(a0RJ=ixK)xZ2ouv_ry))jG~N# ztqp|SM5hr3*_7xEn92<23;G6dezpY8g1QgmWOt-6+h~v1&3SKX&bngr=1YT z^r<-HcTfOr%o`mpc}ozyq^8~iVzR9#nLotc!ERv0Y~W&R%6*5jI{;|u(2B0eTJ2Gp z$2S>-=3vX?UYn!3o@%y$Ing}RfA+j>+n0pT*E_lHc5|^mVhL~M#oO&eEk3#4xYlpY zL1pR&J<%KdFf^Aq>fr8>G`sLH6&SyPVZ+kBsl!OOF9$qFy8BNwc#I5st)3!y5kKkF zdL~up4O~8&Q6gKobuWCEWLK+S^_uVA>iy-v?zP;-*q;oA>F^R~lQ7%se_$H*+a3D; z&2J}eD%UoOv|$N&>vT;4N_XiD(M{vr-hINX(%~Mv+tqmR3tQr-x2b(wIf8^Id<&P5 z+xmiZZ~|%5joZ#V4*nd4XH{lGSIOFxJZmAi0^xT!}(iyf_t?jKTZ zJDi+1MP6w0C)kc#ZB1y%lCUOl*kuN{b(H8beT@0Wy*gu6#kqI!we7v{?Js=q=#Ks7 zSHvInmiL#rxxdlf^46ELb{-dib}=~iBwLDg=cNYSDmP2d;oJRwJRSJ(=3MXzLE*fd z()TM2MW{h-$`!%U=q5$7Np+aPBSO*vR9ghMkwGDXaJ5-NJ|Y?VVACPETms@x*q1UU z0V98Yczp288rG=qPaG^@`uu-TW|sM!MC!3B!t&sM*w=5270`CFxvw zlCJ>!Qx1!2fHhcm+meKh&*4S7ri{>oE-8i(N1P!&0zh-|J|6Vgguj%0X?J*&d_RNU z@BWAV2WS%aFZb?Wm8*+)N#;*A9i-!;KU-6&w@58;_KYga2 z(eu5BdwXB}F53M$24DlLQhfe*|1 z4fT3;_C5kjQ5Aj^j~?NZVFZ`Nlv^@|;;8|&(rZgbsZ;;ymZ%5atJ23)0H^b)ynt6< zi#H~@&1_u=n%OKNrBDwhym$#Op5}iUu=m$OurexM?$sYnR3c6*QKA0sqi3(_q$K$$ zm}_oelh=?Kh^2g*P4WQ!*E%ijw?96D8geqsM|GJW$f4`a^)ff!Ro>@`nI(dm$uURy z8d5Skt$7J=freP}6Va$zLgVOJ$xfY)JX@8UCABh}PE#=e)p@oOOH(bW_5Xj*-n%!q zZDfh!`?st79k6ok4j6;5BzuzmDPd|H+sU}%*IG`pdtE9TA|VM8MKS~_Su>kk_e0!I zct6Rn&pF+V)7_^5PS{a=5|xpp|hN5YXlou4d-2fe7~JLp}u34h>QVNxk+nNm|x=UaVG&E1!#QnL(Ba!!|C7JM#K=947m<_O?{FRJ}{q zfVavy#p=F$r$7Gg9q5%Rv=LuGHpf+|K3GlGSl+@8_Y5+}HP@bw6ghu(OcYXAis63u z4t$@TO;Mw#Wq#W5*9zGwAY2g#O0Y6$aA|4NV{Yd2a`lIrvHgmNscN{wBCifE0{WOP zi`k}3zR-SG&RNqtv8J`Wp=s?M$3SAj6g0vi14ebS8Z)-@j#xA!m zPV9_w5NWD_?5{s#bHab55aR(7i=qc9{QXvuhoekDspV*hOUxT-H_if@|(InqIvx7e+4+Oq2b2T(ScUPDIoo*aA#q&{>HK+o*2c1_I+ z4q(3PQ$6l2o@Re)JJ5Di^?XeK$lCy%M)_Iq8YG=LTP@~rFb(gdsPfF&MTLV?f3pq@ zpGmkQL6xQW9D&dLk*qrjhV*$kKj6%Ynt*9LB^EfpmceVlW%ba8I>7tgr1g=MYvJ~hz5Vc)Hw;r+G>%6gmmg-bFp!} zT%iOt(fMgLFU=ZjO!T)ryRNa|5lsmXTFn-RqEBd+CAm6NGJyWzs=O%F&BKWU1BEqS zRBQU$P|X~cs%D$t;?zljg15n;A0xr0i%)$JL(=7lm-rckAn?i&gpu89aweH@4xTEhK(>b6 zwHh2Kk+vTBRh<#=*Bb@aZ!r}e{>Np2k2raC_=c=WU#Cg>o#fRSh&p>f&g%&j#}4@^ zID7=D^Ggnhd2_R=q0WvcPKnax>2E%q0*M+Lx~G4UC^1>pN?#YnX1U*Ml6J33J&>8R z0`X$Dn4coU$8MYjS-nb15R-2uAvEl9KT@Fp*@Knb8}Sy?-F^50z)SZ7<7%4ukC57` zjVR}vi*lv#U$d-Uvpo<96wp24GdXK<%ZUN;!90F_-H%`f$gn$!k~L{EQuboNw30XC zLO*{BCz|n9aNou)X{RuuEQuzd^@fmi#dK5`#ZvR|gih$cNRt8aI9xrXe_iwaqFdeY z`pvMI8&+e2z5{dY5VAM`wj4Kxn_|=_BfdwR7ZfM(d!0bf2X_!I_(6^ziG36y{SF2l=!H)&cb+IIm$M5Bx{`lMSvpo~FzS}%)9 zcZrz@JE1udhlAjgZy+I9wve|y_(1XnW{(cH%)^G#;9;k?!T3iGZK1O_faFmWI$qj) zYg`|jy$>%Xqr@Z53EYY#O^qc<4VEzqo@AnbU@%9&l)RTan_2T5t&b_9wS2VjT+qVaNZw6qVF5+_%0F*h>>sZT0T zoar|k)$bT};KW{fulV(H_&rB)VKVoe?7@^4;&T@=rFdpCPHu2n&=P+`AX17{CKN6m0M)_bd0IPpPcv00CnAl}A zw+xZ11=|WUY=8M16|cSUFF*pcRhbP(hGq^^-Q>}$@2Hdj@DQi#GgKuVmr41)fZ{=v z0bgr)LHXL~SLzb{AW+YWrJ7B3Y;^ZiOxV>~sqXW%SfW1ovRr|Nm2}BEpK&>dh|MXb zD@K*8w#J+ltLgBnT9toVsntvzR6Gi{p{Tu*hw|jGEL}79R~lN3f83Z@1}u#soK`w; zRlEQ2*a+%wE!N)E>b+S~Ef&0p-B*>Y=Z%)w;2>HkS%>XIb3r_QJ9uVpOH;fM%X@MA zDk`gbYoX(c=8zL$1x-4rxn~`QDY;hRnyQ(&eiTtHV|L4Bi<^JnQOy=0E32~iHD#yusWX2qzY&+&Lrt#RHjgTH<(2ye zem}#RaeomzJ7?dAui49zUN94xL9{MHAPL~@grJGRMOlk&B;FhYfubAXG zu|E1e>%sC(s9wh7)1^#VCxrJVO0oK0ZTcup&L&IP5y)LBE>h~VT+~?Tw@1^eHlc?> zvOTH!1)hKUc&d&Fi^A3VF2cTYqX+&*+s2VN40%Lqo4p^g67`6bh7u!V4wB==Vy4<* zHhs;GV;M~S=70Q^8mA1LI5dI#)x@2x>Z^h_>V1zhh%yVT=9{f$cHl$fqmw#!P|d$k)~s*0DeG$zcm4tV>|=GdgKoQ z9}dw!s#n`d08v71-r|?u!?b!sz1~OpW)5W8=m7xrS6B?s7Eg@R35Ja@$xnlu-d0Cq zUQ8ofyEEmjr68%G%qTk9SiJ8EOlma2+UP4jjM=fjYQ(qXb)1WuMsG{wjctt4IZ1X! zwq8cS-|ilNdFhd`i#D-&GXu~~#r+%%y0Z$x9TgE@m??5TW%E$yE8Q@$mpHjb&J6Ry zb_uf#2<5TcQq2bY>Mh}|U8F(?Y=lfs11sywqS|utE=6_!+KyGOhIFfl^59)NyFOl3 zQ!)#qAaYSbou6OW9C=yXb58)==XtNrCXWJoR7@9RsBj z>I&CBt+FJ{E-Lt6xC)MNW5-HxSry}RCCDJ|jW4o|jX+F9Bk_;WsPQ3jbFxH?-ETJ< z^9*_|^NNPx2V2X0OS->&hq$Jp^@D`!V&qkQUSg%l|0UTrA_8yW@qf4?S%^(p|E{MR;n90zv7+3Dw!jW_TxLr$ic`F_B(Lt zh~wUP`wgf5jU+*JmVL*_J(j_XW98@Fd>Zmxf0EgLv0ZKIz0ESziL8 zFs&^8i34=9AH6ay^ua_u5t+&y!>Ifff0KEAfC9gL~YBY3ocK+8q3vnw0M++Mevr5v$DC=aphB zreIW7yL1U&uw}8DT_-FhD~9@@?1H0z6qD7WrkpXriLD^kxj0orEsMEptle6fXw^Kh zEIOk1nf}9d&(r7UgyyJ8u+Nd(HHXFKY+)>R(kS52h)PTvgz9)>@BjQm=6~TS6Yf!& z4pN=mG#yMw{+9=z?f%ir|MJ=H{onJy{6pq{xj{x3o6AL!m*sU*DEzr})4T8=`FxH| z<}B%p8LXUN);&_e1Wj__R;pQl!4tgLJYk^wVLj9ytQR_el4-jJ#U4KI$ikv%-ITad z(^V?9^%Zf8onztM-FJq=F}PntW-AVa6NhKprhO-z+y@q0e+Qbxe~7%XAZe%=gjWn388q z3F373r|{@2coI)nbCK(u3A9?%^$9)1h4#yJqcx0XfXDQc%)8LySIL+YjHftc(1F&q zy_2u$$>HJi__T-!3F&w2SwBWcq;Fn*_eeY6u-7mwpsBAn^GW^HH5jDmHQLK?Xr7P< zVrW*2NmH@&CUU>R&T{p;i z$PDQ}ON;0+dr{0kz&vF)UV-*OWj3T+TWXF%jY>OR2z47RWGQTa+({lhANnko+uEGn z&%tRr&2|LfRtGER}HQ`xYMLUOjyC z=kQ**HC0(It5>UXhzWIekj(E;w&?dm;=Ty5)9?2Olso65I$ecbq8WBpbsA&FPIK2GITx#)ez37#h8uMcF1N7wgRDKg2TqC zO>r*KUrKd~kpiP1a+8FLAbc*X3DjeAW&)ln;Y8AZa3+H+8SZ`j`7BOOegiu*S-(Gs zGyekXDh*;7N*;;gs-=Ssz}y`6yFI_t$OPaH4!dv=mrOdxG!=6#hu0$k($-^|aPEV^ z6acJfbeDp0wmK)v1VDNujZIU^3yKPAh!Y_mIhmJNWH3h;Rab4$LZSJNQAzdF7gNC9?c%M6iJQrVJzrW@u1s8KBdvvq}m5>iKL6v49Gt%*wnM)qZPdWeSe$ zZGp$`Ks=v-<2CiMIpNfjHW%geo|be}7i-q&hZn#riBce2l#F z<3q4MxDRp}G9@-8-5)UkvX4yLW@6ST1K3 zMTL8#y?aN+d5)_BD=0IP0?75x_dE)X_JO{*?13iY8$op+$l zC`9QTIpH~TmU9#`q4CfIokb@aD$JF2bV2}$^~lv5c=>i1W19(6JvV7vMl-d4&}os( zHJ5zg7${`_C)0K`rG$SSytJoSYn%xD%Qb4hd$DvEbYR{Q`YKEH2sEsqCaDxI`%5`J z@SQGr1g#m+QG_&>IP(aOqneDp&K^W^Y=-hKT4sir$cv!4HaW2syw?gHmUDj$ZPMW* zr8NZ(A;Da(R|6}IbJscT$>`^Qu3G^62vxcfylnc^ZblL^r zAq4nQ%QEWh2@mkkTPK9-w3<@>v{UjC@+i+y44yQp1R!e2ph<6LSQV#iT8t*P#ivHp z?^{RYZbN&wrmHS_4A_RTy>}ur6=g#hTx(67wN)me)~sF3Jm>>sEO}RteKYP_DcdSJJgvv%B(-eY+>9e?ymzY? zbhl2pr*CqX48xB7hLVydNnHvM!TxkhBzggh9wb~0ZZ5|JeeR`Zmo+67>%#+$5CJmk z!AWSOMcm=Q}Q znDELR1xNuAAwbL%9b@Lumx9i$nxvoyl`Fb(EC+dpq04ae1s$%l;NNd7e6vC^)@RylpMm?G zzPdUc5&@3gh__;1`OGEZ5bA zVsooWa#5{TYW-@eP!<3_m7vSs2Cm!Zw_I6YPkpkeF{SbU-p%j-8_ep>tCw%=%~98z z9YL*RTO=$47Y{0!+5h9GEVFwBxh_(2pUfxI>N0I?O4wtNHAD1p-g*l|9=SXWzek|s z7Q+&w>hQ3CMzOcSS@dJA)sEXN1z8`fVb;f#7tTIZl~Bd=!>9l22?ct(Z=XMW{?)@V zYt!J4T*@Hq;D}sZRee4XD6KA4+G9dJ&*J0lODBYQjDevnH)aZ0^<5wcuD|ikd56x7 zO24S=ivgbTG(lZ%Z|v;X3rs!8eWqpdLbx`LqS$DAYf=CB}jaU)T8G;vvwjrj(R09rVdq#0QKuyX8k1 zOL4__l1I95HP(o@TC5aVnShCpMb#U+L;0D3K~uy4eA6H&t0MZ#S#j-Z*m&nr}JK$6}kVkuaAVS}f1Tt~l$46AqBRW+* z-e7U1pp<%R?d?QYHIHfx?x5b5H2+w#bL&uYWJRt@SDLtEVo?EDE|MwDUGq}ud@&#D z-UG|hQwd+kUM?3wcckt1TIZ5-g26xTPM{5c$6MLBC72I?=`;KC?ArK?9PIb#JJ?9R z9V52DSF6uM7{DmgTBlu5ROXwHLpGvH zm6thX&Q>FVP5_SyH#nsa4Jhy#%cdA&6z0&WYQ6JOU1b+#1J)wCMR#>N*!DPoRDNmz zxF~9nYtrbD>cS3^e{Uu(njK|mYPofP3X-nCyd>{|Kfx@zVt~$$6}RKIfV{+)!%1O3 zUz?srA?fJjDPiIkV_x-QdutnK*RxxZ94lIK^%5+_B7ttjv(mKWqY;qtNO+d@J>^l$ zJ4y2MR~OBUEA0CydIlZok{T8|nj1;TLN)3wu-_;vRqqT;Hm6ZH2=~e}#bI%O6bDR4 zVjQ{Ymv7zJm*NAHY6+R1_^xzieQL{fGZL;}<{Ldi-g3m4HMe8eer_%J;IsSY=5Q*U(``X*j_)Q+{1Q;pH&7a}YGxY8jET z9Se36C5N7E=I7+p1$Z`@E=oFoADV~alW=(AI-BWXY3JiI+0D7XG1^wvAmG#@g5HCO z5nxQ`^?pB2mU?e39eA~=xhT(B5JHhD}4GmzLr_A!Nn zuF;{asOFrJu9H-4)EW!t0`EQ0D&|+OJ`a4{oM0F*?$jisabCQhKtvgT%510$D3UV; zQ9~@pOE#U5K^*H&V`r1KdbnZA2S*X^F{U9F;81SRU$YysT5{^LJIU7oez*k?Go(Pz zg9*EM^w|D|R=(ZBG^ui7Yi}U!z(>J|5OrLwBN(yu&{`MnaGaww9!?~1v`ysFK;}Yy z9MFG<-A#iXlJC0j)u5z*Jeb~rLSdJs5KW-%o5(Q?HP>w;2v7p#wlF3ovY?4sSzMA+ zD{ums{~wW=J7`KqsQOmmSxPo!Mmq+%gw$|co%ke?}XaDqE z4pr-~{5ixtFRVt#^@a}G4q77&xc!8YQk-MDg17XpYrS7kF6>X*`1fPA z#@umj=$n?zh6At6437J4BN`*?g^`bnqC=2M6AGCsQU%H*MQjGQJhUJT73NZ~ZMfs{ z(Jg(5peRxD@C^7qC^VzKXt`$2x?vz#cT%NJH&xpYQ7hLK04h@$WZhK+y9|XPL}ug zlXvQxtrYQnmnYxVlw5N%Q!o#s8J+$W_hL&pP{XMv-@9cuVSiTYP>(CfF*?Ll z5Xo8rUq+ttW^BbqE>Y!574cLb1)RkUx6dSDfGi7OVXV+RqJC=fj1#$l$YazZKv6(7 z1kLt;5-Wz{%wCxco?;UFP|Ph86Q4;_QpStz2u88lKb*8*q5;Xw>ibXO(cI^UM#`6e`71t zr?-*b%@(KCx<=ycJfJ8u{Y-sdElkltpEsa?*L?8i>E&Wl9FyA-J!rg>4xE*#*MkE+ z5=PTms_u~fe7#83Gu-FF?N&x0HeW@xe831H@dqvoY5#>uv0ZNpb4_gYe0(cxB99BTIwD7vEdg+@SlMKo)qYa2*;5xt(Wy@#&;uphV=hH*tIG(=vCLqV~{yW8neEsUv2ja_zuUllVV(9 z>WZKmn0*JUen*vywH6aMs$sOSay+`KkT78%5+jFh8NjH)cB3v(as9=r{ycf7_Uj$- z)3Ax}h#@x&>=osiDOo@c@pPF^I#bm&-W5m%vvLhAD|yhskg#y4l6-NxD$CsNlEX?Q z|4KXrbPTurwqv(~B)6!46E4q+%SvtFLS*|6aBP^Gw&{Sw79UBSqoM-i)jXqYI>-mF z7mM7--9GVcunCHIRn4nxsohedATAhPtUgzb)J29sg}rnE$iYVsm=7 z+^o%Ts?VSIc6aYRoEG=KQb%!qRYIvawI}MU)k0p1+Sp;3?OKX5)Mtf4_W zhJdj_Gr|%sz8az5_rM*3V;wcx5V^Y3)ck)lVC~Tk7M+2@x;L7^9XN%(B}$G2L{}9V zKh{c)23)WKZfG}uqdQFCrn8aM>VoBc7eu;@9KV(hot1H>mK?aqKTAKq>x`NaVM=#~ z)%;{LrNfUnlw_t4~}va1`cSD2=0!5;lqXd)`P`HPIobc-w7rrH{J7_ zay%IoPz@4A4FEX?H~Ge>;LMTvgS`u|?kITDA*_!LgMpXM*KC6GA#I04ZrEc-%Smsf)W|cY)mykE_MF3at7rEIkW+8g2nVG})Cq+y75|Q2;5soU#ug-0l)H|fzxtZ=IBzn5rVc-U zsQxRw^TjM+gJm%*Xf_HS3FB{JX!52KnTF1i#areuWIBYhnIY8*&#iurb6@; zY_Ei^izNN4IOxv%FL-W5LKK*Mmm7fPn;=>xh=Ixb9E$)rC zvtDiH69|vacN{4pFvacd?mzetQyj;Ckj;5O%&@ws>+6VBE+_(EnVbeWPIZP~9W$tn zF6wyAbs?t%_S(q)c8u*O+xdtGuAR~o(u3BIeC5E0R%x1qc`LSru zv=N{NwNbzyEN5{=c_{O1`02u>OsH!BF%I+QI*Chjn=|j@P)AzIy%g z(UZf&Cy&R^UVfu~e04Z}{_^pY!z_8WI0e7W8j?f_qpZrJUd&n1;n(H?<*(vhjGMrC zS**a7(p13Jk6+=_+ivAgx=HzB2|A1Axx%~4rRD)vww|dY?JSmKY&JH3`p%PCQJ+=w z6NR;@W9+&5Kb0jw0yemPv--3d7Pa8Rmc6repPVYI}cGOn-piSS_iuJ8@X{%befp?2EIeatgZtGkEfqdeHCVGlDbkOf~eq z0c!hHuR|Z7$v%rgUCUcqKMI8gUY)(V`@;es7+rV4P(kZ<(kukdNVNv&7Kq#RSCFFOuA zdyS8}262|83ry)g@Gn)&)O)XGOs9n9Z8wnp_ARMM1!bL=*hv3>jaZYCd**68j~bFs z*5!v04eE@794jjBYiCtljZKps)=f?_h0_B`lC?BkrE*7CGD8 zf*yAEv@X7{^4MZ*(B_Jt863Drl{*_nf?xC7Yiq7BAUDve*eaEeUOs>I-J2)l7Y|WyNFgdN1867AoIqZ}K}tTT-OP4}9N zhq3H{0?J|&%L2#_#%0&T4(e&8e00WFPDN9Lo7O~}2P&d}CJq?w6UMwlA;qLMoG*1U zLaDeT+^t|JIYr$lf>r{+4(~96c=%YlM3y@WcaFRX zn(~EB(ii=}<0=mVe$vf9(tfYQGKXMb@IKLTrEfHbf4Aj9n#Fex=w0)hPD;A)HyTU4 z>D!KEl0XQ5*IQOvw;+VR-)8GDs9%B{QO5;s39h(5)O`6?fA6Rt?y$|nt0)5xT;S%R z@HI3VuVnCr&h+fMqo)sJsDL%k_%a(cE*rvINe845HlBukY$^!3AFRfE6Is;i=UiOVymlSzP8dvZX+_d&}iO9PbLx<{;m$BIP#B$ZkDr5^Spf+ z?yl{c<9xSAaaI( zAsYRl$2;WiRnl4zuJP4=+5<~A9E}MRMg>I`pnHSpLOKwj9Cn3F79!1n;vsAI!3J&0 zzTuu9c896h&t4UMqo$Yz;aPrj101nASV+wz+-CP@F7b!6@Mm%Dsyx9o?le>}gNz+yWrBDIUHAx{ zqV%WpZP*Tckv3^9o73_)C%U2Dmh|8dxIz&|3rMA)gRJIJ^VUHqPFPU)p+5>$u4k>u z6AF~hg<$*23(Vk17^>`fNLexUP(daoos{=kK_BHH0qx8N!-nRPG`9NewwQ^3&Yq1R zxM?s5-#E+3#4k!*(x{Dv5{8j5&8UkQ3*(4THxi#_vF$^!8_AHE8dHI4x05A1K9ubY zvZz!lMD+*htY)rR>r8DNq)JOBl#6$0a`770n~!5kIwoJ3l)dn4prfCP zRSEUlupl8T!vZwVgnG>vd^l}?h$GCSvn!I3l!_i)71wpLOGTbj7!`Mj&wk)hP>YP5 z!A7kH+4T0Ut`zr8JrAi`z#!`$p`}UN{fHILx-DgDHM;AZoedb{VpPk~+y?Lyri>K~ zLf$+g0}3`(s%JoJ7O;?iDV&sj!S%T97C2N05z;zA#XLQS+~;jpV-0(MBbU8TeVY2_ zEFjJ>w!tO99qvEYpV9e)8@JkK7?Xuj3$uGiCIXrYgf6a{*v1-*cTnKt+F$IQ@B-(; z8JJw5h`RI>$Z&G>~-?hCAbE0353fCkKhS5{1E&uV3BY*LPLQi9z#4kLU5MIxlV9 zKli&f5mPx7TO1ql%>DR7fG16?&>znP< zzn5iyjZraLM%2E1CF%q*$K?Xjix(_uj!}Cj#>vRx5mzNfTYQdKwQ95>++h>r(%e1l>TC!1fwq;@w!1s&5vJJyyH!#?RYlL*u*KDFsh0raLq zR}ew&fn5QOW8hk%?lyYes)zJP|VR_X$`TA*MeaV3@wQ3K1ixa;`uH=+`4P>@r707o`6Bn| z8tUH^wT}7mDyrKwwD2V)(IWjV)zZY=H3Z?1;&`#rC3xvOp8S^*##0#12G-H}=qG}Y zgLiove~R9JTGWB;9TK3)`SqA~somh&)?Y!)teQX_8Z}OQ7>|UJkguyhyf9cvbG;_(lUo4s76 zRBJc9Hjxe?@)Bd-N4{4yFLkPe2BU3^^YBP=E{Z~@BV|mPxsyUzx}-jd!PLt|Mw3U|3^g@u9qy-?ja;o z_G19~uY5koK*70NFoay@%_!fiM`5g<4(|i2VKvh=9H_mmOz%eo>I(1Rk-r8iYh%#X zle5QvWj$F{%QZ0Ewc-iAwtN_cAy1pwK8y#VM_n~fu+E9DZsG(c*{F!W4iA0 z?e~}*B3!;3&V=PRoo`XeU8kVy`1pD}E3Vyny5q1yv37Uh(Hkrs8}HW?Om!xKJNx9l zLMw1S%@)T8nl9ZDrd7SfvQU##G>X;NbA`TtmN$-kq`x;Oia%;uj; z|D2mP!S0&#nR>v39Um$%eQm+Cfs<5!0-k{s6G;xc08loZufg4!7~|+f=WP^6oDOn3 z9vg{;XL*{vvudiyS`+m~qf;1@V1{;lTkFgn1JmJANYS4_1YL@$bYc^8hP|q{EP7g=}b2DYBIL|k-zAF*S+r} zF0aR>5@n{XC7gqv7pujxY$n~_x2@+`kt}k)?K6V$MIZl&o1P?PM#;620H8p>O@TdS*@>2?29z1Gx?q||}6-(}h27SaF0bmGiIHY8)!7y!y=h{yDOkHxD`AC!Y zO_SOerr`=Mh}<8vumY_RtO@hS8xWNWScCRrGh0`P*);h#D(uLj$d#cU7poGh#1@lv z)1x6Eka(1CizXoirS#}PNe8*2KATx7M*V1_!N%ad2Z3FmFdM~tsvCKKF(xb{8gAM# z($>UAXu&Cu`#K;)-fMy~ec~Rd4VsD>&#IGhay?PJ$=PCYUZZ*zcLqz#L$Q!#v0PUd z)n8DCs`KPYjga-1Igl1s%xIC@q8va7I6QM52^PIyi>s?|kpY8evx9Ns+2*=rK zHh+TiDOc-j9_)-Ivo$AU@o5u!2j(j`daC*H!&wP-2v8CzW_wkCUZ`V98ZtuKd-6I= zj?48`Np*Bi)YVi=sB?blz_x=R`=mI9tX1m*jCj-J`I}b?P{210*({sY9xS8Ze$Txw zDIEhAY$-1;iU-v7KZj6ZGouQOwV{gNexD_n)4AX?=A$_a)x0%@nlVeL!aX35x1s8BcssZTWp@9www zA6FneI?7st*EM1mw!_;453_k|_uldad#|K*%1zknvN-!Yw$fpY{JF7=J=(`&R}JIn zr%k{cft4MlfXl52vrN^@hxtc2$2z@MqI)%QCWT*63}rfh_~j;LDRl%DZU5yaA~gQt z&D zN;Wgg`p}MQig8h{i)pbg#+&676HyH)cxC^PK_T^VcO5 za5uV4g-KC*hCh~2H50-|*H9xCYbMlGF@3qJ7EWyQbaQcWovajsgpQK}oS_z*H7Nd% z7aO43CWo#T19kceER_SI4WeILAb9lT(T9jrT@&fOj9$q9~}v=*K)j7gZv;U|Q-i0fR?` zS(}eYs4XrVlCOQYoJZjgaZ!x>o$zO7Xq*WW?~Hf$K(J<)q#m z#CP5(aQuD$0Esc8(d^Je#SXr~cV$J|9*CDf7F=`Ov&CWwo|1rv^dr*MsK!2C;&&)Y zKgBDgTc6A$DQ?ClP*F=MNWmjP`7N@Hkj};%9tVz}ZERJMD`J;*gX{6Z_u`j-yAT=< z`rn7RYE!-$e=?>Swe}L!1{YdOV>lbFi;nB&+7s?rDc_KigG4b6>$mlKm4Ua@5f4P< zzBjy`kO3JM$L1XkI=DkQ5y0c8PTuz!yx6?=$40fEGh=Nl3MbC$ zKY3BMEmMs3G{LrQU4_C^c8hv{d^yq`9I!1SHtCSJ9wlC0t50wADvtJR%h?>um)sow zxR@x0yB<%Mtp%ZSu;LMZdDNpx}*2IQ)a~D31NO zw6>Q$WNZbE)yd9ZURO#8EffYTblq%pcw3yXRBH-M z`B<*wm#KseqHicbnxM+E(4SI3g1If+$Y2vej6jg6tvQ7}R|}JW-cZLi>4{r4P@a5s zolF<7+z-JwNCnwHW280GCFBV^+tCS1LxC0}9iPP=eWx$2i?2jZ&SZ%Rs`M&%*(Rl#jB zvFlpfW|EgO^&Af;EtlY^V^|+?ff1#dBqxOKklRWtY}?`)Tw1cW_-SMqU+DJ=dA_X6 zwNbPz6H>LPf&6pE@=M{W@D(%G#=9N9UYDqY>-#p69OB}CAoK%dt1VbsEmZs&OK+0S z7s(oYG4vkQgN#xGofWk&aEP`Hi{>#)HVoOQGqrC{&h4<|W6qq*b&{;O_j$jYcn1sK zAr_vjH^q!mB$nF{sF-FW65=@oFG~*v8$~m&%gXtAI4cpn9M+=+d#L>uG-m_n7L)D{ zqb5KA&qUsTU+rfkwj$j|$C&ln^#J2~?}hXBVSI_Qs`htViVkzj@v4}g+a~ZMxaNv? zLp}7Uxm;PB3-Z`tMFpbACMby2^z5NO6|`p%VI7=Pu8x81Rz1vSU`HW`Hq^VRi%VB2 zFv{j}(!Zs)x`)13$$V(S#iwbw#CEz)p3prs|KLG?HdFI6%9S9|sN%V%)k@I-NS9M4 zsFn`g@|JYuG1YiXR1ekScakT?N`XoF zzVZ)$l4n#Q8cV}8LR*#`?0M;&UlG1&__2oTVp{N8;JY+Qi?dy^IqV*7pWv}ApIt8{ zvJ@K_k6>qF>0LSqu^q@5{q~@#Bgsy|obaxQj zzTY=a_==!IY~AIP3JVuVo(fTeo=C=fkzaX+N4@QVa1=UhC~6XH_S&!7*=h<5?PUFb zEP`Rb-nN^J7uJy2GtcW&bdme`cABj-n>FB1`2w&6q?4q9iC`l~8Z$#suH!I~gLkFd zIOq;9JaGn5e>{gtmLJE3=B=1>4rQY@3Bcdwk^g`b*#m*aWPxy@sAwY#x6eqJhRk-Wau7Ajc{yMf|HSE;anI?-GdKjFds(jGSNXvDc0tILBzAD$Dww9!R&@bqB@!E4ZnXyD569zWM=fsC99a zcI8eoIa}0d1gR@EB_R0AtFfe={Rj#&&A5z3vst_1%uK571|mRY4ou#Z$mXZR^ z9~>l5sEh=l;>u{WwGJ)cVvWRqOMUVQpQ0md9WE|P48p_~HE_H$P?!`6q*R}kLtU%@ z6HYoKJo zi}Qwr09J**AnW4u*NazwPm9b>cEo?gloP$13uZg~p*VB$;fpaRFpZftrd#_%0R|;IG2Ej4duoMhc*F0(zMa~ze}JWSpxNgn54`~5&s6}r{!c1x2bVj zu}?VO5hrDTMii`}_mxhG42tDfclUr9f>Tm|?|+`0m)A|<$Kz{%Xb_YR3eq(-f8RC9 z%DjBtcK*#HXTfdW$C>OQBBP7HTGdYjDeVjy) zL;+aM3N3wZ*6Q>kcUn`2;1W<)T7ofUSj9eO$jltJP{SV+gWF_I1}D!u6|>HAWp z5k*Vfj@TzFnN*#z%dxJ-`Y@1ERmE6gv;P986SeM&py0G{#fKJSN>8@Onofwf&Ljo# zK{Uzi2eL0QQ^`k*%82BF`Sn5)dh;(5sqkEtN+v? zanKB-)gdminkxZgep;q`S!7!Wf)5r32Pfqv_>1E(SKvs=q#*KAz`7bgLf{3YFY$+2 zkjMK;wW`inMBf!?`YgXm_9*bS#*|Oabs% z*V2}ZghhvN+ zCg*p)Lm<@$EgFvxBD?J*x0~Fyk=j1@de8@9{%cQ!`CB8-A5NJ25akai$Tug(AA=Ah z5#DP{g?)1}`e7vWqma)?W)(U0NWU&R zDuMogBSP8|nOr1aw57KiazmC~=X}~13{3Mj&4@Sgn0)k`SB3Ne zk-?UEh7VdqH_9?hX@un*!>Myh5h~(ehTqA5$IZw`M-*!qJhwf$xdT2K6yb*cdv3DJ zW#M>SeM}3o4A39bklf2=EF0jQqP~$(q7O6nM(q843^**}j9F6u*=?u%@V-L?CWUcg zfo3G^EMWmhcVYx5-{%|6y>-idn|Rh`hW({tJ|nzJzfe$-*#96XT3r z9qLuha!k-!T&M@>?tm{36QrpI@JlzUZ4(#*F-J2HRNP9vtyj{Y{+LClY?D|&Ve6c} zP0w;Z?b{Da&GEjJDXkf2$Y*!>{19}p0^($^DmtxI{XaEncp2u>vLm|f)32a{|L`S7d) zj?Kf1;B=|@YyCMm&dO01G%0+4*yZN3BZ`ZV98i5)O|GDxyH0bSP_Tx^)ai?6bU$5f zHLclVC1kEX$1mbt8Z*N$jNm`)t7@DYDi)^#36jSFfjn#G;tnej!t%Ynmu4Ua$ws=j z6OpNC1GQN&{JO`wDe`_-)1eO~$k}_eHhKVzW{_9`+gOt|uWiBwgRG5cO$T3ASmnJ|oVY z$nEbsbNi&7cZm%VP^hp8?Rq4{(rkY}HIF^*NIHTThZmcgUU?hm_e(9@?DEYv#<}Z6 z)CtJdCkuY>z5CJ1kT@BCJqOMwYRD?1WEIUiL`Ii+k<4%{4Yjr`-YBtp9YCamgRu4r z7CtATgB7EKJg2*P6b}SaO0w%HJ6;vjz;pmto8@}XT}?Yfo{a-H=aR7oQUsN5k4^=Y z>VM43=-B6=H3GCOri}k+du-Lq58RhuZBJVF!vjsd5xZ?0AVl1M{GD$_lrfRF1OBBd z1TCpn!<_ixS43Z%P=zEm>#hUZUIQVVUstAoxkqb;e>pXNXJ&u7Ti=ZSf`~4Y@E(ct zMGDx_FBVMOD}i}aU>>p*NM8@_VNiWhs!Ok(DteBhM-p4-3SKY3y=Ol$`4@vGHW`L$ zcE2mn{SHwqis@{BCWT1=k3ew0?azwKQm1Krdq0CZP{pKLUmxwbtf+jZ5fA%F<55?@ zX!JX@BN*&DPe5keA9SwSiE{N8eT*f!hp@fttHn?au0The;Rbe%ny4f5B1O7Jcei0$Yp$ zk=!ED7Lrc8)te*QPRJ$cWHyga#oh6&Qg6%ZcYolpy za#c9@BW8v--HJ(3%ZdSMTWZTX;G4;429#}9%Me=Mlm8d%&aCAeoSzUwI; z`aV;>yO1Jvt=xNGzWt-}><|G?OLr$@9`>IEeO{jgrfAmT;C<&|#jr1uUG8t6eCf?* z@91U|si2ziBB?JRRgQ5tK~gQv;!EM&K_*oAWzhP4p)d?)Ebsqm#r$%EAkWT4{*YL_ zN!@V5kHq9Gf36gjUYv~g{Upro6#Q*^-{qX+%wm*H%mNe>J`r2Da_Q}SQnL{*QtC+~ z{-(>`er4?*GEQoHmk4sNrNDJWKbAa31P_admxkojA`xB5yQzgH_oUWa`-u1+Qrr8D zJbe89!xxX9j32*#`1PCdtCxpQ-#mT!;&59ZlaJy(f6^FpT_c61MUMHO+^79L%Tak; zttlbo8tZN-4Cb}s1NGJ_@BY#jho5Lyt`vSNQ=?dJwAWfqEM%p>xw)d666~(BzT7p%WB}9djy2?; zm{ZOOf9rK^xva@dHBXAQi(GOC=b6^r3o3#Q16QGD+dF$?!S~2vOG50SM?=&v>qGpf z*w@_84?S-xjlyEMRYGHSl@w}1v(H-#Y$}3z31->FW<6Y-4Doh=16f&9*-AQ0g#r~z zcL^+=Q+=(%Iz~{U66%%!qEu(DwZ>#-&O-@?f9+QLL5H0#s(E#>xk&kOW-gqe4O%cN zMO|*N$!;m-$$a}-o7JYwdT0zn zf4gF6N94O{WoW^W*vEv}ox=78Hk=4pkBmo`XDf;}j|xYg@kSC+8ruy*xg(B6qAlEg z6+|l!N_(sIKgitE5xCJ?v~)$Ak7`JY1&3^Hkcd#|QFm-en<*nI;tG*Ju7FD^KgSU{ zS9iB`4BLz;<7|8R%;H-3$enQHrufYOKU>!% z&KmLiYV^l2*)K;)G?v`q-my@8RqKp=^I}G^e#zl^wWK(|rZ$|Bw&>pHajEhje^g}a z1&MBDa$Vhl2UD7{goxpripc0 zTF#G1`t8w>AEq3*p@JYDXWywFf9UBQ?WRYUS_q7ZtZRtiuwPp5AgLCjx(Jac`c;(@ zV~2YB)c{C0@z1E3hzpi6mr?;Rv(~q=?Kc9pJ(`H){XOX~1ieILH(Nb)WBb;|NMsjf#S9IF8A(TEmjJ3+&f)v?g6}LW#_%=$*ic)f9@F&TCDQr z_153mRsVeY-~s+${p0?BZ+CD1{=Rue&-Xt2ba(GxlHFfp12&+GRGfl)2RiH9f48a&sPn5DF0XIjI$15C=HhHtPKdk}$CE_2cuvmoy3zU? zbB*bD59imw>*-$#!6`YuP^5inpCMD>{>tZbY=jCI925XQryRa!&YsI3EuPt5L`7mvOfa?$Z1tC(Uo~}IxgmFY1eawi=g)~V|qcn zPrsoWT4pB(tPHKvzUw!ZGat-vQLGAu(Pm>)OVWlY$OAe9f7rYC4J60}Or=>sz*7ZhV$vRv5frDeG<)tD!MhK)|Tqja*mTI|!+>Xaa zGQ9^uQtd$Je~F|2rCcrignKsS97~|>j)BVBsiHq6rGUdf&8J87zi58I=S|6ZogWay zPc0VAt`lAMzMSTMX)StzRm^Qw@vjRdHM$>o0=m5wWQnNPNqJGJcIEp5$}uPD@nXIK zoAe9RWDaz`3Z2AJZL6Q2&}P>o11;ogS0k(M?tv|(f2CBEH$?mbsIl5_NE*#pNcc(9 z-?Nh-^}d^;*7>HMdwa(-Xq4U2XOi<}j;!n)VvxuTwTh8}#LJU&!sd9Eez8uvXmFoX@OpD%`cY%G>9 zDLE`l=4e5?>1Z^VxJ_?0>lS1TS1@L49k*_Emh3n~?PRF>89>~|Uw@8OOmw2rtkRuFMgy;ptq?!GYT z=!)ybW}TqdF+VD4q$+@me5bv#V7LewgFVuGaid-{_O);CeGj0f(`Dz$tHlx!k6Od= zH6KBQnG7LLi5*}q!7gM10XXocWk zf7QL6IJWI*5K$cJq}Z`fAM_(}}4odnU}>RjDNFnnWC5_3YU z#Z}Ir$^aWOUPQt0nmQp2o7qgCtSQc3e}KVnMp0Q%^<7;ob8V_M@^Y}s=IQNg8rj%EruHb<-uwIgw)gFB z2F9d3&odx3=+N){R(~B~0u`ukudWK+lF4GWxxhfT@;xqxraEL?FS%d%Lf!5s`aVq)on;xz{miOjXbh-HK&AfjL3{I7-m^Thsi&nk(Jo4 zYJJU)$Fd5bc*@~FsSz#wDGmHye~rGQ;W)HzNIF+GRM6A2tSZ9H7{yVHyOdoibP5E+ zG&XyTl2UNgS`uT=!?FQVX@IL@t~fA6G$XIKsfh%CrvteVlOxKJNTbJ+_72Ztayr&+ z!BWhBqLYK$?aPs%ugbrB@$~DLub+>vW-nfuMR6KWpFDcQQD8lS3}-cpWTuFsU9VGL4uCT*qR)#Hb>(JVO^QjNOk^|L$j(EKtJ3x>SA=F`~{M%W8J)8yAdIs@@ zFvb1>)8n%i-sv8=f3}#+b57MzbinN;;jVOhD*0QeYeuPSU$5Wde_UPo(NSCdBF)PQ z+u`kj2XgI@-EwWDb;?cTyA`tN5w;qn)3=^}t0JnZ{+}hM!19)m(F1Tq>R)HfO~!PJ zM=~aheLPEcUMvEB#@5{1%%@xr)HamS6~w+|Bg3`iy;!WD0%-w1;Bp#O`xz~*yxB-5 zmB0Xb9Fxb7UTO7Rmm1E_RLP;|nfHv08}2i-{mn{9bQAFC!m{(_V#z6~ppBy1aK`?P zx&wV-tg5fkf5Cye85sk#P+-@APwuUD&aZzQ*W$pTwVw7M>sHEnn-M60eEtljcA2~L z?L@^gK{stWd}8fZ3DqsO5ecB2NWti)KsUM$KI{Trz!j1kVk=W1b6qF&`?#DfVCUA0 zoJa#`d5g6+^A$<0_I0Ma@o@<})(`_zkhdhU@YoH0e>sP3m?{4hiD;Uczt^Z=ZRJr+ z{Xp>a?e_+xh>1u}iW#XmmCORWb;S%NxKG5Asa8Yk5=@0NCAxkeIJctvD$GW^u_Bim z5z-wTaaPog1wzyD9Y9guu`#}3$F~%)cUI2|BjxSmq#n9ML-BesK1CG#*AVZ_<_CZe z<>HP+e_~8=$A-*!bTfAWv5u6AoXu@@)|^e*ZTxdU>|pv7?Dp4F)H6|RO=i%zNByE# z<7bJx_MkANw(ca}vi-!rkB)FQXhW~&Npx9ZHQEj!A?X}pYbRP*)PY?2d_5qbNgwEm z`mpPt7^)APwy?;*RLe+kf(N*n47Y4)=|CPxe|Y>eXM9(s<@I6`?9R^4>r$aAMKd*N zMGjDPUn$XY>MIWw+pE!}p{zSQZb}<-#@^a@VgTnWXkrZ`?KIJjzXkFgf7=l9pp8sG z!)D&y2pe$dbA+ygR%p*~D$Bjz|%ImVkFdTw=by(x2FAL_>4>s8*Q3^*^Z!N*4d{S_ZiyS>;h+DU+i;!{F|GX{0=IFGzIsf5BsN zhi7cIxn?)Rgz!9q2Rjj^4cMJ5H{;C`>`}*{F%;ubO}-H6D~>&j+zg~*I*qA8$DRI< z2_oUbymmrkef(6Ib2W+iGM`V2)vCCTek%0w?@{mv5v)iu4jcu7)ikh^*xafu5AC zKe5<}h^#5_9bwNh30!L}s z9Xw7%)phc|&)~)8y+1Z8!aL-Huv}iBB75yUTa#YKE9dnpy4T*lZ}GkMQ5f@ggWkv+ zM*I0(7}FC)S%ls)oWNade=aVrL58t8G<>$DKChO91Qqojcel5ZL}~^bbxdzVD3gWK zaB@`*+1u4wmEzah6iVPdByVNNc(ztDwEIg|b}Op*J*2@27uqh8=;vTw*AKqQWh0fz z@`(Ur{lT87s92@IG?k?Vcle8HE;yq%S#u&}LU7258Xh6%6y};Le=Zl*Gka|kplL9gzf#eBH z;Npapr7z}FK9(z0YhkrC!AZQ%;UDUpa`9c1h5nSH-DpeY18O%iB|PiU7tDVJw!&&Y znQf+Ezr1cH?pJZwf2?eXsaukXxi-7W;^MfPdlIt1*V3(-6zeqjSf;4i#c1fKmm0Ks zJ9g_Q+CnF__Kx4MBXph~r=9H_Km(bXyY?@d%s; zP44+#Jk|*nJfW*%rofE{Sn6bp71Y(%hM_s2egFq#FwsK3f4G`6Ck(5sLSP103|N7C z2Y57+&J8dpBRNbOmyOF$F#lCCladCy?gt1u9q~zN#4^}8HwtMBWG?zwHDQvOL`!(gRn985`j~_T7g!A zB?d*uk}3B%bhjmU$1RM55yXs4r{GeG!@>(JYj2|XSg;jnJ#;*YYK7q)Jbt8yuN;IT z?DC1|U<8FOX6p)+%;enj?e_`u#;HL?OAv~yr&#X=f8Zui%D9B=W^NC@1HADCvkqXs z*H<9sN4@JrPl;AA5rZ{jIIPXPQuJ-k9k6(-Cet6yMWZQ1WBJ=EIi6) zYxLquf59&0UPn%aSdT9CQAZqNq;VwCet)mYUI7CwLF6G3a@+$~<*=F~sx~f&5JheY z38Kb4dA(7rhYq8Hl!rAWTXZM`5(USQR)V5Uw=&+1SP1e`VeXP8)JPsvyaF<{!13wO zc}&v0*7=2+fq_@WSi-Z#Yzk;jr&}@nr6v|vf0RPo2x^nr;#i4W7^Mq*Q3-jsN|z8~ zZV+-I#S8$>)D@wJ8<;+{su0`UwvfgHh7SUicuN185 z8IDob>1Pp`5{=O==OV$G5`nT!1ZoAc@EDQfsL}Jq&dal?Kr(u}&sJf_%iy-ykHgMJ zcgNx5bxAUM6Av3o>8x%7aq=dB){RKMf1`PS6fhOn4>}-WF#H~3ckmunsECs8P6Kxd zN1n5ndRv>-sGsp_w52g?bYt^YA1$Y7>e|AJ)fyW(@Hbk{_6P9Ok_<5Ih>P@Ohl4CR z2)68KyXZ&{wS63=$W0X@+bKgF27p~lg&HYtS_;DUh}?}sWjn)V!SM{@#PMm`e@w4k z8TxUQI&F6-f(xRpWJvvW6g=Ufw(-_I)jO^AorRoUsVSiH5)+TtHhKP7sze3PH8K%O zOEqt$Ldx(~s4vAjv9s_R1nf@aeajH8Qb2yGhL7}WQ~+KBCLRZCEuFo)Lxy#eGlgs@ z2m?-GL}`ioReW#d`EmK*FGS|bfBjy0ZlRA{w40SoLZy@jA9+*44f(1O$&kBu^pLZ~ z;@k>Cj83o-_){lmyqedfm=!BMpHAUDsFfb0nr*VWAexJkxFmSHdW*w{&tE-zGXDDY z%je^VFCP8*)YX-uSCmzf@RZsM$uHVyb39A&#)wkWUDbO9VGH=&eSKcy_)kgpf1YK)1v24d-se>=&ev(5Y*tj2`X6hIs(u5MOM%0wHy=a}G-UhC|MT{DYd z$*u`9%uBnh)7!Zaf?W6~%|e>8RcY@0#y@9@~`sU{UH#aa2{jXerdot&dGc~LRYzPwlEmSaV(AuFRtamy>#Pa#Fpgj7XpouI4yzJ8n@YSLMt^uHQ*sloiIfpR3QK(~X+LMRBe5 z#MaV?n(yV=bzMygJ$;+F5jJjBg6}*-2`^ z%A7OLIJ48(A~mdKE@P0nr39ERfPLOxA@|E>Wf@Ot#b?f5u&-p!XA~m`Dpa+@mpY#S z<|~6NlNv`26sj*k*Hq2OH~`?Rj__FZz=H%ZL|-T~qUqcqrvoj{3!qRz$j5$^4Dg=CmS9o%{d$KmS*KVo~diTLdHX1ptW7%UrF@U=0AH7Y$?t z1t6?97bP6thD_+VX!3(uE2x8Ax#sRRNp06#XdRrfs=&nnk$-Vqm#a%mGqEmax`Ls$ zz|WJiSj{TPYKm$nug+&wCe!G^VAmhB1_%v0fDLaMf6f=-vV#pJD{&r{3h{r;kirerhExxUwzD%0os;Q2T{9tiVf&k|>$s07kTH zBM_3se`K;*kw&{PBY6~v85ogj6crdbFEMuT-sx;{tPp|8?CY8ffAYvc11c>9;HNJ1 zZ;zdaS~YaQKNQbc5n9p>iEy4UTG0cY|I|V=eu`6!pL)>?0jc=)i0#eI?1jr*U^3(l zD!VF?grf$2AMDmftB*SK5OiklY=ZB}(-H<1fAA$6_a_fin!z}(JfsL*lLY*pi#Ay@ zX^Swi6vA9kQzR)BBp6sDI*qL;;*;;6y#8;mUV^1ibMhE&$m_WYyCvT5#gqSfW2zw- z@sbsYZB7Y+1Z;2x3BXdf0KK6pKt3~C%)>XvO<$qfC9Oqvjf z;g&ScA}Q;!3cW1?fM?oo+Mb%*R}F>pCmlk2uR%xMcE_|3e3BJ}jx-s#*|)6Uj?qz(?vlm-w1oax2V4-!NRib9 z>N}Ibde%(CC4|Hhv!<|Vr((cNQN8f0W?FC~3Ah4m!jvfXlfAwjYEK zjeU$Hj(3Z2Hl@1oZjllttaZb?KrP8COi#LaUfq};eLe8AlgP}dzY6H-e{Xd`oy1^t z!(KOQaq}@ko+0F0@yIH8VEfa!1#tUgel|?zn^}KNn4N{v|2yejGE4e|@ODmhNT3X9ol3Izm?i6pu8@WLjOOVIP5RwR-%$^$HHsX9h=U zkPgPrNSVc2NFjV~3s3ZkwiD9agw!i@3WP<22qj9D1qlShC_ynWur(G?j*;_Y<2XpR z>=?f9+%Ie;&n6P~*BqVQ_dsn<& zB*ZQnRTm4S3oR(2mUPq_V8{5vQE2jDHknZv-RRh6MzI*j26c>@KO2+Ezx@8suka5f zR=#(+caQ4P-#cAy?y&;*_Xvsa-($tTme<>UV^{t2>4OLOfAx>||Gm%l{%D`k^S#eL z{cQJNlHFfpe*-pv=+x)`NzeaIa`@`;e+{20IyA4#;Zv}IRwory2Yt8%WAt!8SIn4h z_H{KYL1n$FPS4hF6jDD(>`$b5tLhkPt=Fb*$>p1MSg&J4f1u?tZJD|fprG?M0F0pO|BT56fHEG+ zDR%mVHd!?_A0L=!UcXxO5pr)0e6jh)U2~L~pO;N5&60no)A8QZrktoRt<#-hH9y&L z>}=Xm2;YDD0PJE0G|7x@-dP`X_af`%^VJK@;;VH~0yw<6BY)VUcjNJpzm;U?ZY6jF zf9k&09kzD>#>uARIMYU;k%;|8pvoe9FYRt*W8oHNVgmLsn+(SXjiZEv8wtkLo%gyv zJsdxN{qXBIPhWg9db8U2;eiH3N9K>%K}Ss30#WD>d*WWU0VKC&d!ZJc;E!kAhUJ`9 zI>8zMG?{_AW&qZYVMzQ=jK))lG0#2ke-e7<4JY@Yc^XSpU*rUQ0xfX?&ZkIl9`Cli z+xvxxy`dMHR>*X^51GZQtv3Kjlg6e0*35W&nxDGV86!|UK49RVLllwI;Ttm7j2x0i zkJc?_Q>(P79heWn0zuz?r@487VNjp0nT{Ww-@st=hOrB3f+7I%#&`;HyFJIFf79MN zKBv3xva@8UDK8P;U_J5%^`wcxXaQ@{55I-8l9$GorIBE~=T@i82KH@$6D5Y!S{m7J z#F5mOPt1k%i!`gv(A7=%cbsY!V+a`-k9D~(R^jP@33-_=rD<$3^lKoL|Fz@sT2Z=p zkVHbululV%(xYif^6bR?rS;eoe=T!*iPg01MO-JT|M?f*?zl~Mj{MxeTJvUf!C~E1 zTDL|xVID4_6sf5`8mt}u=}vPT%+M~Ofo}Yo;x5+LX-nx;A(2AwtYgCdw$xzN9I{-6 zRVu*h<%ja`ipB@glCt8);vwx|7OP@-*2=B=*=d^SsPe=ClhAk~xQ z{e6=-1hfPu@JELo0lSPjaMUJ%)u&oh9T1aynGZFsYEtrG1RF>*Zgw<`Um)IuJUQyd zoB3NlP{aG?L81xwOkE7_pb*3P7VEQkfdS`Kf&Kt{_Z+{cu znt`LJ&0$tE$5hr#8*_41f2SM8N~VLDkO9kYYuKKdvR_up4=TgNQ;TFh?wgSH2rr>3VM^xoQ+ti{jaCV$u|LQg>;5vi$VHn8Zq9l~=_|t#U7m)qx7d z1&vDDt^zZ|A&^VZ?W|`MR>TT&=EgSQ<6W4jX!Mrl3<&Q{q>fiRe@SS&SvLUKepiam zRJ2sa8tn~!D;J~>%>G`w@%Pe=ze4H8k5RMnre*uU(^g!Ni%62qya1OU2g}+@5}4n! zVV=^Kk`FoP64vD$)D*mOMC{8D&*?A=H(sZJFvQP$E9F1&AZEe$uXQC_JGRh3(SJuP@Gn7O>*tzuKF^n&W7}Tok$h9?e!T}E`ilz5|&k= z+f2)4Iahl-xuyu2BALOSU6r_V6fq0wQvNm(-yd9`ivIM$7$&8kOL6f1*Z*NP5>e?Zwk$mJV&fwliQZ zm1--o0+cC#M%8Szlk`xRd?ATdkJzZx)v+#C7e+CM@qV?b$(tCJn+98Z5D7Th4oxWH zthk2SZD};&0HrMWBsmOV(8LNJ*iWjPW_Jj#&5T_rEoiG_;%-IBtUQ5j9wLg!5}ksi zpoDnue~0E?tj$M{D|HOha;0EV7b3P%WX4!JyLK)501pW4I_UfMttJRqfiC1G&XS92 zwOU}w1X{sBU2YRuaCNq@1$w9qHJ-R$+3j1$#pL{$CuM&4;xQd&bkA}`cE2%?H^ru5 z)@5pj))IPjGiNt?_tCxFdJTtcN$&XWfw^r=e=kV5g+0|v9`V(tW3)?+$F6}3Y0FLR zTiE$;Y2MSog%6v)ph(Tk%a`I{aLjh8 z5ep4YkwG;2-rX0DU4k%`rsE7p6(NRdEer!bIJ!)ARIenAbT@E~^5Ie?ns2A9G5lr!7so2OkYvzE4sX4fDEcQ=!M7a!Vr?h zCK!}*_4P{~CPY7an9sr(5@{-A-0QL;HYDXu$+P zzX6lBTSjEMnUvGvrGlAb5KoSa;|fX^RTC?4pxN?sH8N02OlzfoZxO>e{Ku6mS_ef5^#bfj9~aZQEfO$-wV`g4<_{Qa%x_s%1RIK)5KH zH^%F7;N6$Tg}~+(LHJkLkI8YGUXW@LTy=@ofH!5m;KH>{7B&1*w^2B3bAj1(QOf)O z*?ZIGHjX4=ct11Z`42r*>>4crErQZ`{lbD{blFy{WNAgpv->dkLQSAafA$EM(Eurq zSO5K~%&a5p>TZy%MF4bc=oO2R2v z1@W(j`n~AY0aZx9CPAoJfA!EOPwg!TPSUr-J1#Hb2cBM)CI3+@LD7NbktIWFZM3d# zmrsQVXrD|CrPQPavd{*0C-l4!$^$mw8=eNrro_DUZQmmgUF0(zXiq0W9b9*$2!~lq z(+n;u%aA6ypgRIw4B+7^hm3?jLW-Z~AP)yNZ1=GoFR{j>BH6iSe;~C&Lg};21d`nV zRG3s(`W=-O>f)fZ+g$twavli*EYz!R3x=m4SZKWptE42wXMs`J{wf4n%=onYC`g+r zT`$#&QrVI1Mh{4ODQO|obw`rskmyT5-=nTZ(iI@?sZ^i2L^(2ho=Uv!pqcE2(1+Cl zlE{}YHJc!*nRHpre`FYhtXuEZ>f~>TemSsAx_(d==tV?FjG&2O6UV%|vR;8@J{DN? zSakKCD{KtQku;;2(3^lHhu#H9iye3d0`xad;CZ#H0FjCox*`e|>2@?w*|!u)KlXZO zI5Y8fFRwAwe5R~)T7l4K>Bod7R{xQyX$%72(2@U+;BRj2I^|>5+>VKsJ7KpyUYGt;L zgyHBMMd_A$f1a2t3jwhxrr^Rlm~utkniQ9rC}qK7H9_)5jW8U<4|CJ{4Yotg4lkAl z3AsJur3v-E)^`@XFd!lEssKkI(g!)r_jGZ|%irWxQS0bp`!Mn~uyZx6999w4ASK!C z^^apC^2X@w!qU^=33M57yD%bA6g6w#i_hA@Fz_@Ie}(8c;%*0tF@iHh=20Q;G`hze z9ou+M*2AQOy4F-0QK(OV8w^?assEiJiQqv}azPSw;uMtv)LTmQhOU&|`L} z%ne91f4lGa60t8GfacJsHUdB(w_xn#Gq{WMNr(UFjED;i%@ID-b(D`!Y2PKNpIRP6 zjzQ5qU`+@if2gh)e{Z7xYN5>+g_a+*^r9wgs2c!I}^)rx);WN$eY0uog^0QU^1W(gJq ze<2yEzEu3Klh{tVE%#C2Z&_E?(XSy7Ew z<#G+N@N0wB9NKsV<$Ce*yUy%xqbr}b&^)@Zn*v_%G&cRa_avFicXNFE&%Lt^#cHlPu_6tMoJ@ zvzImQZ&BB`=$5Lun@(S}8cC;_aRpb$V#e_~`cw&M`A1r89$ zQuHCYqxdgE4{`phqGz2qBE@D&dUz~#_a8s9dfS%f5Inwf&^1D!0CB&Sdn7VJjD0fO z-pIDj_yI?gmS+)mIZT@u)8?XwU`U_VYtrJI?utjlLz~73`gVKfQF+B?%!q#&)bJKz zB6$-Z@*8`Phy2E+e~XpN%rtwAJL{od#0EsB9a4&@ho8aVy=^yHFaeR~e6s`|s2a^? zvf$95$?eB~)3#<9J}2h2yRCn$dj`0K?4NhfR5~;ByRVK5tK1G>KYQIdJTIOv%ki_t zba`IRQ*mnvf0AWcj0>#Vu0bPjfNGE)Jp}M3*zThCG_v!&e;R2(X6{wTwU5~S!ruM1 zMiF9}N6(%=ef{kD==BdT|J!dLo(^ya)=d14m->Mf06B3uFA#AHg3*lSIKHad<|vbTx{08#cA(E2z--@oULG%(he%um^Dfj zNIgazNw^u_Kl7k5UfE>4=ICvDvMTZqXsj8ZgYQSoa1;68~eYRR`!r5>?e_UuU-@tvr^yftdxuRDiAsUQ_ z98q2LWGNDMLzm)jim{a0NARjCl%TS(i{+v!;=D?Ifzw8V&(@Y6%Ng;A%k}J;I>}HqvEK2O zX)MYaFB2Ln6>81RV0GYt+_Quw+O=@o)WHpjWU<<jUyt~)MD5yM2^+S5q1?S*jntYtsWsJDn=7(fG$gLa!~8Dbs;70K;ChPMIAA(D zfAa-6ermmImCDpH=XvEmu0d`+QI9M-_5er_fQW4oV=l#uobhC$Wc*-O56Xg2V-|^K zH|0;feJ&P$p{z%%3ySO53c0av@Pz=srm*p>(}pYVZcQQhBu*hba7ih-yj-Q;Ub-SY zRehB*{4%Hnpdz%oiBpl5(6B%?Q?VMFe}|@9lehy7mZU@%Tw$WPLP`ywY@~@|Dzm^7 z3R|uoQ0x|YTzyTGYUc%xJh|?qJnnWx)*x=qM?Rek_Z*{AE8NL2cBZMHW!8{E6jXMLhU3E}WZQ-73$%5MD zjAj&QQRB-vj4YnFk&bFZ2QiT^5Vx-Zg5~TsY=hkJ$2scJd^xP)RK({XGK9I+VeCT) zhIn0vd-a%MRd2A5CK}4_o?xJJf1Jk>Z~2LGsOgw(OdT&@w%6jdB(3Q%pd+n`&Tpx{ zs!<|=A0#ML@z;)-Qreup%~djWT3_9^WVFfQvdQT(Cdv+5cw@#rGKc=bb(_*_)Od{_ zx7JbxomK-5%S^)meRw3J3z7886mPvd^V@M)=GCQCl^AO+gAk`nK5_!Mf8#aB?)A{R z;tB+2k_mX$b++&)uD2-!XVA*0c-ZJ_e3P5QF=tqJJ>6X%x<7E<*$u}*?SsrXJVf*i z&>!u|Fsuy}8kkju7!b`34(nLOCMcGCJ@j3?n)+Mj-CBH*5oP^d+&3 zY~jzLEK_*NF_DM3;zQ@kf7id@mK~5|tp8xXdGY+|^S{41cpiwTx|8Uy%p}?&m)c^I zy!kuuOWG_W1pM*&qQWHTRXHg_0D&SeFGmO|3p`%ed#mNc5iWQWe#6xy>hMCkJMgO$ zI6C{Mz~fIVe~wG&0F_M#$vBs8W2^j98WJZZ#4}E>vMts|kK5h5e|(CTweesvE0iCT zm>Q^Orl6+DWeV?}6gkm-J0F8j@LKyzu_{iY%U@1gVsh83x(8syR{i1#8ZX69e;Htb**#k zwQTghs-^o2RJe5=Vp;Z6@`^PMvBVa`mshX&&$B|10dS(tn*EzlQBP^C)}((I&nUn{ z4tu^%IwktfA$SfB-Y7KpK@E-%<&v^DVzc{Td!alK8v;&`e;7>H_NC=a*bkWS@on+b zM!a=1F?40QXZ17rN|Fs=oC`Tn02FgRT@}Ujs*A4^S^IKHA@bcuPxlgilO%s&F*}W1^}sVx zsbFe`zhHt@v>4{yLF$VA(-=hvvi^2tI5se(uhUAuf7@DMLI)i&P(B@*Z6BHi280{2 zV%=%T*58dpar+O1#X_L*$&B6Z)u=OB`tq7e9MB38=l0vr=;e+ z+wOe`gx*&-iM{``Drz8xs@cZEZji1yHLNv*9r^IAJc>Z&1DPb@g)*5`lg`i#nCi0a zijG<@fAuGK!k>N$U12)+YXJ?fuSvkZO*;!>x`8kvgnfyaMQZvl*S0(rQ ze(uR?H6HQZGk?FA^u07Y7$iKbGiC;2ifFOYXs0vNVuX<$O4Pw*ScaK8wVi7PXGGF6 z)&?sD`vLn3i~w+4r-5Nq;!E`?+OHqWM1%!$e|W$WFUATcv!J_4JKnXC^0g(k^=$rD zM086E-pGE(U7T*hkE@)xd?CYnUL?lNgB}C}K6Q4a_ze6Ul+JMv8>$>_>JZcEwYF!qvIYcp|9~>i zf3d(%6Ar7+J9N@K;ZSYy>e^SwnA^2aA5T2yCibZty&G+2k@*nc8oj_Ra&p(^<*vCC zv6eM?wY}4;O=E>gwzbvU^^8=)4UmOL4S`mJ$W-H&56DUwS6j*#t<=HG%v%rBq9O8q zm@h8p4RO+3rq=}_y?Ew;@|uS$*J^CMe;p0ndC&<_PmRRHM`97rjv4ElVY1gXV|T5y zNkhvzI4>o`L>E-bkWj-C_#;&VhHcv%6Muv@2pBuXr`TOB- z*^E~HPZVLz`GP94B5&H=a67&~-6k7eggt7_Cbi~6+L5yckB%S~KAv2Fb%fzzf5jN~ ztR3M^?hS#dhX4B;C6vkjQ3_P`Ee&eNMpT5hqw2=G>mM!GO5N?Shu#-G3-62eQ!*b^ z4Ws}x`6&c0%&0aufaJ52VlTK5Lx-YkWld~I3=LRkuG?lm2lUzvU+uw|o}Kq&ItPCf z13T}!oS82H^>!Tx{p98Ar~ffJe|YhKp9`5DYY~aBAD5U;M?Pl)lB@+N8epheJ`+E5 z_EJ81G;xRqWTnAIm2Ltc@PRXcAWFRBY`u^s?(KSF*T3u+I~ zMNREc>$3G$p#Wce98WhOgW)9`_sgb|k;jw=6lfZ%}U-T&p?vzk{bY&Vi72B9mo~3xBz7a$5uj zR*+Gw<3C#l3A<|C(2YbjkKrP%5)q7kJJ`;sx`*r#hQK<0lk5!~OEvME2BQ1?NRXO{ zrce$_G{)ktauml|;!g6EQXeUyAWIH_5@@3&F4PhjvW8hjUO#VgW>dcf_mp@BCMZC> zyox4zs*r#wSeNiSp{;t$nSa^USg3fCD%E$JTvbkn?0z?C)Bw@sz5QymiObz9A$xtC zcX0PN7_X$a1cxVy%0W6vD&Mu{%FyV`hYXm$L?I1CnCY@xnpgF1BOm7WbdpH>8B7#} zkRI<2rnO{MuT{!=M2!xXy0NNa%aS;$pt?eYgi;JrW>llzAxk+X%72vA&-kOQ&>Ae6 zbL+(+?z|SMEE$S1V;r}r#QIS5z6a+b%Z2V)&AI1K1&yiGBX1Dby-4Q0vY8i)DHo%1 zChQ)8?&3g0_@8ajCJZ&m5A?7?It>^y@c9aEJu5#4k1ZErP1Y7uxaTtqJ!SLI%MFWM z{JC)lPlJz@yG3rVE`KA-wHVI|FetRf{Lvo%n&_&N!e;_>HaiD<_kMNkHS(p8zLwB= zYv@}KHG(@K)HSnghr9YIRmlon{|)t?libW7LbvXf0%c|06AX1#C`1rzOv>4;n3UqJ znO<=NA};s~NSe4Z@2X`@ELY4jNTqD%>+)_(+yIcm^Ht#Rd;;9yK$0xT&DpJ`fvL#=}nI8;U2XP>|y8iM#uH7 zoYvbqtZ%)UG4U%1-8%4rmCn6+FDyI@g6BOs{sh;c8h^OXV6{e&BXbec<4&FTm>+84 zdVWS*gpA~N3q^dw@f>wS;P=n~jSfEH1{pC5oxKzz%fhLTnI(~ohCIr=g_C})i5Wb$ z5kVVVB(R`9s37n)ID9zNA>#HKN>CL&^ms~$NYFu(Zg&f6vlU5ys8jmyINO(hd)Ut08KQb<}p{$>Suzgp$dUNKMu$);2 z?UfM|V%$(zhkJf@V1tRH|MV<{PJ{1*4c;9*AHDeD&AY?V^H)!vKRdX#y^)qg`E^3= zlo4-I&T3liyR9&;n)kSCttrl-P7# z&71D{aVw{%OOzh&!Ez4X98h>?yQq0ZrG2#2=e7b1;>O13iM*R&5GutG+G*)d(&5C% zxp~RDoEGHFCU_VVSA`4BJc6OOAac$Bv*w!rr*$3VMelInk77HYLbXj`u8xy`(`4d^ zf`6L^$2QHyP=b&aDfS?m7Wa~ltz%)6V^g?W`wDY#nvChWc5Pkl&}@cbP9ln-S}vnz z5p_2zvP1*zZZqT3dcnMRn|;u#DeuJ^S_a_us$J(6T7eKRdJOeBP_2UWpwL5MV5$+b!pi>49qb)JkI%5`QU{V!^Ax0ykUKmJrgFofeSppjfH;&oW@s z3Q$DiLyi}85i2nXOo!f}tqCHPm+Rlt7)PAt+bNM<1gE-2xxs8m56G33rZBFNM*VmU1_(rz)|092L$EunC|GXDu4bq zj#`cKPPj`egN7{-#Q;T7EG{#}K-C>Y(>6!ke`^pfBwWU37Q(e{HmPZ4{JgKf_^_h5 zMO1P~`vN{mDnFjB&?w#UBh7hUBGM#|C*?)QNtx}An?|}>#W%ef zrM7giy{7UA*1`4pv5+=r69f9+hEUe25)W}8X90i)4E(PWAU7mVZTk)cZ} z!tK5vdenh6bQ#v6OIjOkAd%ePw#CCvfNhlG_V2cSKCViyjktpH-Y=fD54XW0Qnd>3 zn1Z~o-S`hsyD78IPe}Zytbgg`9L6PinaEHG#rm!EJSEY?MXP>OOzuQEkpKYYU`I_X z9I1&<0GEn2dY2))E)&6{uUNij>GkPo#AMFq(%Eb#VYsU9CP&NE_{UsN*L{*#@O*ms zA)1(jxf{llpg$T831u)QF;@`qKi<|s68k&?r~zLrT*EzV(^cpf_J8uI&yz2?CRx?Y z&!2uAxHC;27p!)Rb0jT&HCj3vxMGIOv6+Rrr!5(;y<^3y)Dsl)H{g2&!&+Y{; z{f@`M#@@Euw=Vy(rhnsRqp`9Qh3!L)B-(M%yG7Hl^l&v@A2~LJ1I#<96Fn1&)`3=; zo8k860k(m7iq_Axu{R#f^JR6uSQ|TUT$N67JS@HuOcVEMleNJyWE*h}B3-JmN(Kz2 zPQsqj9bQ43<=_IZ(nmR^>Cd&-d~=d0BBvAtEPs#2+wNWfsOEkgE(Gk( z9A4LQr1!&9kR$TE=8@4;MvziWfP=KFeZ*{CJa&g6OxO{t26z7kjRXHXG87 zr{j++>6;bwZiVouaz6Y<$*LXC;X5OHIPGmfP0K zn74(E%0J{5_J64G0&BwOY7ToVqvI_Y9P19by|ppC12yKiXnYp2%z+$$tu@net^9V1 z@~yz#rlI0mj{Be6jPRN4E;p53{f>+|Y$feB8Zbge3n~&EkdXGXd$naj*D{$+5=VK;&3TtEW2fk`b9BkIuo*uEAinB3M$24A87?f{1vQOpi7Nd)z7w zyzOA0-z$aOBBCg&Aim`|p0xh=V1Is5C0=UYmeh0fpPYW$(II7v01z(- z#Et<8&{V4(#(p%%gE&MZswk>gthEjaX;!Z_L`Pg-eRTa5VC>YbuYv(-zdlB-gGc2O z`9Ov=Pc<8I*3SsXqhj%S9Jez6rq?BeXo>-bc-a*X<-S$ zjUKVy?ps{kk$vN@sgW{3j%$crobH5`3JodM`KM_Ac+&H>sM0FOdRVF8xeWHdjDM|n z6ck(hFaNv$K&|xq7ybK}i@$>x|~+o@@k7Q_Qanr zA3ntYia+*$`}_M39{g4PM!)a(`~3(1lI(qs4cGv7isAp4p1)wV-#q)T-b-=&1iRdO zF#&JN)3R6%lE+I>Klb)B!TiW(-^=99whpoR6x2s*+Ah-#iL)me1G{0 zg0caN{?AT1TP}bjhldWHb#y5O49tiu5^^P~*;J6r>7=+2b5n$q_qph2gteK#T3N#; zJA-rpQw4(lCGUQ|I-BJm0p>_B=935eQLi;}c^!e72jU2-%}hw8vH-T?`o%|L(ZDN0 z7LZBuk7YGfqA7LbnRKmGF3C5^sefSa)}7s6IX~UCf(rx>wwXanT{EuWQbnhfH`~kh z?B`OJoyJwt=uJyJn=VdrNGf$E7b)47#aR3b>u2J>{*7J*>u2iiwU$3S9mH~20s9Nl z@&wIk?1+eYw;K;eE`)rhOJz<(juosmB9 zBJ4h4?G`Y{YK4L;c6UwbsVbN8WpiPl7UB}Eh9Nw$kgRTA=4gUbvySLmv-8kUB9(PR zaQB=GqL2@rrCFzYw3f4uya#HHcN2Ld=qP%uPnz0Sz<_Rf16D;;50<9A$$172H4hrJ~QaJAX->yIZY_}DU^#+|4l7!$mBA!s^U zAUcmuXbNyd*2H@3V1Io?_9-f&O11~g(dZ;?N$ceh#1D3ElfV+lSYP?|55W!T-Kfhl zRL2-acJ;RTqdQQpBpHz{0BEkPhC+IpLGnZXLvpezaX3K_p4q_coQ-}Gf8EP! z#@oyG!Y8LcD2~Uz#$5+gmtEs7pLadSLZ_f{Ja>bWw_*mi=lOpzUSli3*WyKf31(m$ zuv;?=x5^>@W;ii9gjlU$wxJey(3P*Y3vJnQQP~{=RT+TfvPMLRm?8C=m4Ub7!wgI1 zI|2r(4}Y2$&2bA1FQ4;lJmsnb=9@^gtwkK4Easa^y5lCV@{jkWObMRM0zqejXAqNE z%)!nEszgZ344w)U9US#5jQKgnP{3MJV29vPd7&(M#<2>a<~iOf27W|k5v$vYh1bv? zF(hQiMQg%G#5$7iR$_0jbQSeXJYZ8^>JCI;eSb`=!aoX&D%`g6%v#!3dfyly73ZNa zc}_Rc=LR|lYg$V0(3R+?95aT8s|{Dku}WppgSBdmM!fbSFRANo*G`!cjLZzMQfl&B z`4`??r4?f8@I1U}cpMkx!$+_TV`CKMkBGN&3S_>Qwb7MzJ!Y=Aru{3OFPrzZam}zi zt$%NY#R#>&6?|XA)2)}0T~w7N1!^$W6{=lW*OakcRw|rqyFN#8v%q~Lq1zR_YkEhs z57%A5Ho|y2fxKBL4+-Xh91pZMyuxzbO*feD#+W$_8mYM&_VjO-FueX!`xiSZ z;6&3+)Golg-u3yr_(?8PZ3`aH53k3CEwuHgWr*aRE7ze79`?-cHtoDCU#)09Y1HXu^XA1ONr9 z$L3`Usb5#5trZ__F&t?b4f4-owWvCs6dXz&U=Tuv>U6782ww}29pkzUv#fgvA%7S? z;l-^os$snzu$O@YA#CpTljSrY7w3!Vq*$pWWRR3~USO5?BFV?&V!0NgFhxUBac=bE z!Mv{=8nAfde6_kV@peME5dxxl2uRw=EOwHw>Ki}Fm!bWR_Lu%S68|4&?zS>chhNH= zeFOf0G~A`sZqN2;wI zs$yJxn`X}A*rsj2p>6Z7wX!p9WpCQn?$pxG)ZX62|F~~jb1jtAvx+IYJ1ghqY%}Z7 zPxq3JO(cx{ivBI=vmodpnLEUya`H*GtYzBHnq<3kZe>oop`Y=e?DwBOHGiMgjMgl% z{0=Qu5xUo4z}nU?^RwEq!d5Kca1+6xUdOlvb@+%W%;#fB=Of4x@Q>Z(9H`~uL*-;+ zfI{_3{b%uuP~3Zfl}yF($zAmI{`fy1M?XI*Kc(pVU73Fi5}szsp{!%N4wvqOQIPNK zoPY@~W%UX2RWbiUnb?p!M}O~iBH-XgJdfJlKrU#_YkNsVLOgcHsMM?miLPX8vDwrD$`W9?(n96UL6>{6Cbj4$&_!&c2bG zZ^OY~+g;LsQVcXDWOZJ+f=5F zb1&7TC`O8!$rxO*G88z#I}(P0Sd)sR3dc=9p!~nBh%o1Z%6T^BMKt02A%UbWeXY@( z`V5}>C8l&eR_}?daDR>vYc;O&z;q8y|7v^D5w_{{xrLqor9Odwnlk=Cri^f-s%Vt; zJV`Bh%nIJsTs$W0axD0v`2<;4poC7PnwzOOZd>88^zq?~*FXGeGHPiu0&Hf3DCG7;l-m$!s(;ZL`5_LuWTSX1WojA8 zD#gBu$r7!2sE_uU4?5BdIlDZ)0%3B9j9xIw;dRVYr6)(o{ED|_odStfhKO5Iv{Sl= z>Ht>fi_LTbMk~b+a6?lyLSq!IQO7%|34`H@kNx;!QBHu2RprUFNM65wlEB>Ih^i{9 zUAE#C_qz61+kc4){Czb(cpg-bUA2Kg;fl_&9Fw*}xwMKcDn5><8&GSU6ysd%S5mHn zZd4U_c$Lnso8@9IwI`ZWL{&nmeyD6+3&G=ayD0PIy+YpikaKfgoC3FuWB9A5ckz;^coF%b>?kw`pAqzR5g$>t?Wk)11IG=ZTM)itQ3V4J=cj zSKA4Emmju^ONfS{=>ZoS$}`;;lqbM=s>~^tw4!f#Fna@W|z{y zPf!+b6n2DZaV!A_WWUaBwiH*yV<;iLe4Q(8uV3Ro>#{kVC2*G(@P;w!Tqc?X=yoWV zRhS0Wi9v09)RNiSJXkkzUfI%tWy_~(`{!zVXO?R@w1jR!?Rha~yQW>s*KT{qkCx6K zk$?M+SGbrjW&+7=q!-JY zH+cw#)RIc2J1x_iS8a>;D=L@NtnW^8VAMBMmlR&-&dWlu|AH~KUYZB-muA1;&-(w~ z|Nr)*ylBkRFBs+>cKy4t`=9r4)ZlgtM}NRJ+-6&cKzuCf=t45OXd@T@R9F=VL0Kw~crJ ze(HQoW8I(sv?+xgR?Hz_?~F1rlFK9YhYYNnlsM*59FQ**Do@OBR32A^Vo~(ZD}VIe zBQ5TMSVTzsWv+a~F*5{YHKR zUI*85TZuW-gj=?E5V)ypc1s73&*4|l)+NC61;1%#B4vI5_RW_MKaXi%5(nEZ?%{=3 zg6~%}$Ll$(j4Lh&!54LHCuDyVKY#6QnD5c8oHSA1s<@v)Kjgv8Sv>?4%70LFPqdi# zPIv~X9~R8$r##rDsh30Wjk}PueYz@gFcVX4TxFhEV~MdEKX8dsO`7(Qc0I5;w(1d8 z>wF{pdJ;|zFl8;W=YGNltf3z0*fdVlCNV=2^<&U|5P*i{N(!a)@;=1Eb)N|SM3aTfEAF{5uP zOG={kSoc8cg3XEXZCA;eK7Xp%>Nk@s)_R7v$QJ7&YMTT#zhJglOG4EzLOWp~l62N~ zdfEbr>#gphT_9O;tyaNvC?fN)zQ$$cwMeA<{yN_%2Y&6jk1qRSFMt1U=Jq;3w6!l3 zt>EMewMPgr&64G4kCMX*j0f)&&p`?5D8l=8c`mNJAQR{KkGwLf)lv{_XFR|V7!wACzr(V_lE7Lc(~+^|(C&=H8~ItQ;kP-ohxuEtd3z7X!2^pZ#jO5~OC3&fd09)T^g+k`5*QXVK=`+v($4?DZ`a=XlXO_hNT zIddT1xqob`H75s?vZ8{3;E-}zat+}fn{(hUolB=Y2Uj;IM-9aQR;$~lu-W5=WBB6 z8$&GZ6v8usoqw4NycG=6KWQayij~QtsC2qm;KU$Hi{K$jC$6(4hSA>rCA{KpO`6DS zT3Az#pm{E;d8* zPh+UJX>)9Q{&$}{i#O^w4K`j(?EyVK3H*5>ncWvZe1AtSv+UdBH*a44_vrBXt2fWz zK0bW+_PN(WX-S6aFHWU~ZBoew{P^M;P}7DReQFh=yh|wV&Pg4cww{&AL>Qk*V=h9T zE;gz~j+8q3tf7AhIF=aVcdNq;`d!Y(=flCtB=moBPlvo56e7ExsTwlYdwhQXq z!5r}ksge$s(S{!_U05pGd?;8YJXwY%pDQ?m6@Tw7PJ6haa+j!DUZyk-&;jSw?dbh<;V+*tM+TJY2Rn)bKCE z(9djxWBhHrSL#kHUi2>n#4Z2lQ{m6R07WL>$osT*w_$`Xw5c|y1I&%dXFyFl?M=)B zDt|kl9U@`51>3L+iuv;2UW3y@EBI^3N#?q!S{1EZh&_hq<_MLU^sy1iCm>#J> z-=Y_YW5MR(lvjedtZy8?9>foEff<@?s$z7y1pD#HVgX4CSQ8hSxMrdfz1L zjXGe}+(YK(uW)57RSZOD0!qDS9 zFz#u!!!RZkuf)*43R!iYFN>qSW5cm%>HNoZ#8gdRfs$8`zduc;1qDa5Kr7L|g7fkvNyPDJdIf)4bsZ;$v8x#3vPt(#Pv@j0eT2WD=OK?Pj z_vteHM_%ck*s;?!{Ec}~S|EVib$`LgrII5W(_##huXN;t>(2%#N^<;oQeId);=<91 zih#))#Q|ZI&~kBEtVWB|(SFBmnMM=Sc=u&qT#f|UU0<=A?O!z}6epAQ)v_281wto` zaBWV;sNLd#mc1p7=CSz+aIO|6^x9AsY%36usscxCL|obZ(3RZvi@WFMN>h;W2v zzLGJkVM-OB)^*)Ub^|fwC4WfSup04CFsXtG&j9C%U=entf}a9y_8(+7s(xrbjlcJ7^6R z<&-TzDH#*iR8EJeBnqMyNeoSB3*%h#U2mY6t*K@+<=lpPHpO8^D1TJj(7ZO`cyR*W zmWVeJ@-2wDrwwqvg-L3ha$Y}#Wf);Uo!)KfC#Tw%`D$4W``uQw#1#Q#TFgol7*1xp zLRJx!fC`xKc(FT))LgC?Yw#Dis?Vq(>xWMB&3t?=^HZH}rc=)aP)Q`*4+48LFAACC zDo@7Kh4@Pk-B#3a9DgDuarV+H1w_z+>mcRjY-q(3-jFwpI61Ybny>}4ShL1%t5(r_ z)`I3X(cSj6w*~#(iUzl%!`ss02AaD~NOap!=z99Q4GnHZhqt7~Z4w1E@Vl+}-exKu z<%8Su!&?N}wPT%c=K*2FBM=E4rfk682L0CpJe_IV1W$412Y=+Xv{@#b`MR9qv%xdy z*-lKPQbPNP0)R$NvHO|jP~qkzj7sSZjvGy?7u%xm4`zmN;AloR>LlI3BKuHn207+7 zeM7Kb3hWq@?O$QFJE7|zwp>N`#uJchn_ORL|Kll7{7wFa6cfL`t|rFXhhL8HV}O}1 zEuTaFF|HTr#(&9Vx0XQGe>Lj_Mm0oVnR$d#c0_6b)=e?yv? zws8yu{fX4{EH>-qX02H~d9Na_)4wFk_}jaW^|b4n_#4L4xHsl7{-n4LcIcp}HrDp` z{g*%gG=KfMm`-Q+FZ%Z{7b}76_s^D_`yg13Fv(+aUkRCuRkpmk?ihRG&zBD$;(x^- z=f4jh>_7Zc{YJm{zkKl5{eMaJKF0=Zz?>w8|6h9kcanoQ&;G0TvK$NgSoB^@z{};d zgyM{kmk=V<+lQc9+3eeLT0C7WuU6&R`T9_Bbbo_H{}ZCxNX@-iRZ63N@$%&>i0FfN z^8f6h({X+>PRiMGA!zjDCr`WL-+~I|y5G-+8YREr|pPu3`-2_>Z{Hj=0 zBY$WT7)n*C*tww^;SeN3Pczhm(9PI0<;c)A$I)rcfNR90JYjhww`k?>AoB^IEQoWq-aZ zMicQtst<~vAlqIIjB+j*8!^rG_*{W`0Y5!~U%K+|o6}Qq{&665yoh^M=It|J?$pCW z8v?z-8AWWWjxv6ezJ>c*>(@B=-D;H2g`EDb`fCw5d#0)D60H(~eSOzZ#93z?c3cq-RWKxdTM`ExpQ1)ZaWPeW=YsF$# zyq)FoY=usv4>!pG-Q>I!tmPcVoMWcF%B%HAeOmht+!j1!fsuwLAKL)WHy>TtQ$g-6 zkL_bSgdps!?+6k#Q;hXgGHYGefCs(aqLw3LD)cEh11>&b73gjffl7)Y*<7YK-F|LO zaRK$cL12!L>RMaWB*AGZNekhmp z9Gxy!qvrOr*$A&6ED<9GeB^U(Pjqr+1sNyASrx(2C>nu53eXWnp`U?jfZh^Ar){#$ zN%m-jv~eV9A!ED-7KcV@Fw|btzj!?}=$^6OVg(ixXSJj5V4cIiaDPt}>1@aPW*kWr z3u)BC4(LGRVFNS=0?}Zw1LF^GO+zOU2^JZm;n z!IOr!#`)}S47=re9!?kCNU&5@q`xjASPuaowPF(98E8aHonjh2z6`zvcEj-YLX;jc zfND;WPIr>O7dk}<@P8;Ei_rr_fg2=0sc;R7@<`-XN3sjDmS0pBFj;qOQUX^AanKri zJcxw%$6%Ch@^YHb^5jAGAo=&bCp^s)u9Q8jYVkfEWgwohmK`eu=I56O=YlT*xZ1dDlf_{MkfGOHAu0)H9a&c`Eo?vl=-U5V&41yX(B}_LGg0vD$Y+r8TKhT_^L=Y^IUATQ{+$$HiQEgj{`A{ zp-nnJm4E9dyRz4AH?hNozI*wq_t)$}4B?PApY5tR6Zd+hPZ9;ydoqa+ihCSh zr#Od);~xXr{(tU1ycG2Jw2L$nfI0`mHq=g5P8*2%rslWa{If*Gf>eet&0dILo4{_bv? zp<{CeUe>O2ww?udv39&+%=!15uMYp|uIlQ(kvveV?tj9WhWx%!4_ICu)x!o(CuI{D zN$IhsyVEoP$ck}B*H6^Oc=;4#k5wE-)I;^z8;K_ybU^or`BH5vUzQN`h5X1iMhHP8 z{iDe#%E7Gi`G>8*KnL%wHF~Cj(J1A zU0*(Q21tWUh~|m5Sx(@48;?YPhSxiM%KxF~EYpEBOgiXJ$4+5)yxhbhUf6!sBn$8y z5XA#h&ofJ7m}2~g$*#_pSFOwOu9w*Jjh%B==YMHLh1JA7=Hj8fC>NWGihEfeiq>It z6A;tw_dr?6#iToo6s3XvbyC_&^xjOT0iW(zsm=!E+8W6sZ(gX3W8gQ~j*we0q77|X z;A;eL!+2!W=}L@OLU*H94l;#|*sC2^&q-ejY&A&c^GpclLe}wHpxS}V1)y~mm|Yd9 ze}6mXDgX^2qDRz|qiK;yADKf)CmERxBIeq`oEno&0LNVQK;Za%(+DQkgTo#q<=K3( zDvsbEJ#{R{Zr0)t#Z|?5CI8S#B9r&bJ4U`!-PmThRFD#H=j#|h)3IhD7zPGW?kt}3 z?{a$sWGH|*_*G}#!I z5G`rkZfH3|ZiYGARkS>S;S4OHu{-`mtt|zcp^=v4W(LKlh(Yni9E>l?^EWS_NQDWa z1|@Z#eewubg>c_XCh|^T+n7NJ*t@TA~MORf(p_U6-)^Z zb{Bt#41Qybu$GnNxN^{*zhY^*Geh(|oR*n0e(X7ErDBc*MLGKtyHI}v`)09PUzr`p zCxJh9+J0)R5zfYv^2pk=FRDa)VI<1ln2viiLHS|p02u^T`ZR!+I4W;`_2{_{$?(f_nqUS_tD=(-o(|$ zQ(V4gW4gH3bP&=7Ox;%PHc7cmc+NuJ%t@!yqG5B3pEkwnsvgp{ccNl!AGOl#9piJz zNsmz-b?M^olhPFNynl}T_YtuGq0iEVBiukaW&Y28osE*QmllfrhT@0krjZ)`e4hzM*+&$_P`aYjEtF>8QEU z3IsIxA&C6}By0_CO0R#Mg7n=Ae$ z!kRHmv%@T13@;d}9;4!CsJFle7ub_plycFarnkfOAK8FD9u1rcA0zMc5f~qzs!4(r zlpKyDj>3wVS%9n-M}uxM@E-kRPstj2^=nO{&g!a5>e)jlslw285a@=_ zZWwvdowFUfXC2D*hBXt0&M~ABUwDv`{m>L=K};?gdOC%M1%+qI{4l&*(XA&~XUokn zz&f~;*{483!euu*?luz3Vc@rJ4e1(M|LTTs*MG2v{qxCJw~4tL2FSVX4bX~l4y2NT z>;ZlUe~J!i4{VU^I}9uKyMbDAlCTaf?%ml>PIHIWzq+;RtYJvC=B7tPL1{?;>9!-2 z4P?*Vml~2Ztos{u(lvwv zdVlh%mAx?IcFzN*gT_LJm#cDJBxu^olWKD!1bXnl951GunZVd02fj#NpotVLw`w>M3iRG}l0CK#3MbQf_#p z>{zY9<6!>SNJ7*4tey16gh z!NeOD2133yu-XRCZdlW|HHbRlR+J^I!fFwBZ)jIwH) zlk<+wFsfIV#p3`n#NH13C4cC>FPq`tYeE?;oeK6~AFX&U9{&KF`Sd$er;!R5B}=Hn zfSFr*eWk`U%5p38jViC}!-P6dtKtIqB#;5{+d3DgAgfJ^?>r%TRWw77P{ICD3r7 z$W>J8r&5s7^KUm^&3U1k4{WP$1?KZvJgatKjA*eWZl&ig%_XV*)pkFY*!;7+5G>kO-_pW_+|}u3Lf?oMZiAtdu^V2%;yWBgjx9^L=)99DO!aGR6w@-5pDjY<{sI>Yq6XnM4*H zeAZJL&S-<9Ciod0!i$f~Abgdtwsp&X>z4bWmV7_ymw%^^pMKA?z=nE3GSBbczJB-S zAoP{`#=k@FgTu#%&qoI@Umw%V(n(*Y$AU0haFR$IJNLPD60$_eBvzstHp zS_581-h4@^oF}N8M*@-ngzz3&d$wa8Gpph4dy>hL*#P6Nix^$I2D~1$>(qho(+@N? zB=!>H!GG35qYDwVoNZ>Ej-FHx{HpHX_fEE(?4>(0vo6dzQRx|E@pnpc2NDIU_wgFh zo1~pj{hgy@=X-~(A@qZKJ!05npkaQ(k&s$e!A%TZSh1PX1RB~r{N!%m9^XF4ZNZ&J zwqA4qa~AkoU@nIaUG4|_Y09n{Y<*-i7y|Phv44(?19{}$4gWSb><)T5P!&M<-(z0# zU0~i3eUfdC1tNy&80+R@nFE5gWGgu`QxHv$ofGQVWy;KX#U>Ab{>@y)*c7OlMtpa% z-vGXcKzfT)^EPU97n6F{O%x}Bg<#mNcV7**PlUD3T7f16UxLOH3_lC4&L!4+zZ+i2 zXMa7%Ezc&kxd-G7Mzdlz7A*7T%ZDSud=>J}fO;10ICY!|^fvz@790FE>10JO$a^i{ z#@BEDyT#Lp98Kg++TGp#zR01Z!{qAzIxnXQ&5meX>rSgZH%HRSj2 zlc$G|tBh#(x8S0-6zq4>dH-IlOne0u+JF1^VqN0z)j4Dpsx~LRubw2xC4tQ=9pK86 zy<~AYN4||n;?=rHFvSrXeWxX41rl&T=7a8_zY{au>-{@5h53I?lLc^9av@~9#Y*g+ zEGCbO0h2JP=7>p4kfN4CqT;I>_Qh;LJj|CB>6P*X%xBZ0M|*$xSZ-AYCC!x;ZGQ~e zPgW{_rdrhoN|RmjLwFx8V@6jgd+KMd4fg=Z#_Ur3J-A9IPB zzrI)}|A07{smwBVs!>hI6ojxjpP#_KOd+{-(MBaabn+kaVMQ)EwQe9bQ~$7=sXDoW ztuZkFf7UY<6yx7hu({SE=6&h^J_$yF6kO^;?St`@83|B^&Y2uhPI+aC^N?ZPVL-(>%EM^b)$h{Y)|` zdI_vCy&ifiQopy|!2T|gf%w)Zg7ysXV&Lt{lDFX3gH{MwUrF`c1Tk?)xAgpl{9;#J zl@6kEZ@O44QE^4qct~ObH-A{5yQXiGugO}1Rgck0crovxCpk~82gAz@daw?d-%E%u zBw~3A!MJe#0HgqV17;N*wn)I!)B-gts?2bFV%Zw8HLeOyKY&Qs7ItXJy9s&VE0exM z?iD_itNfCX7^?)S-d*#urNMrzj|;Br?mmcM&QHeDRw5?~b)bj(!hcz}Q*`Dd*gXsK z)-`vX!MkVm>*}3h7zNjiZJ4Gu3Vm_&(wb`|pqc%FYsaCfYj{0q0#&NQ7d70ywIa+) z@Uo_+W+S?KtNh-B9rqr>#7i+_IeaR}mFjN$L6SSGy7<;k-b)w+dH?b1he?jLZ%$u| z;bah65@_I6G+wcX;eVdFAa~i7&|N~e%)s*7H@^YY8>u?SF9S5)hD<{*BJv!)ZU9W4 zxn9g8wht{|2tBEL8ch6!(A&Wp)A{P@#9j#9zEGg z^J-Ln`LKh=b(6?L{8vklZy=_7B}AP(p%)^NsF<9T$elw{9N4owVcGSPHF}+7iZfRp zxG)Y7PWsEEq|;9xJxacOkdm)C{oGSOKMV@lc+g}Ild=`aQys3mQUdhJ8svWSZ`fiX z^BF;0{{!6tM}H?}UUeH(_F)YcSA-j-o~zcUzSPV6SWJr*y z4WnmCxRX3P01N8{=y!CUfh_sIf&=6MY9tGT#SFw=@DL(#)C>R?K-k`~;DOif=<1`~ zBK>u14ad{hKl~_ll+Rwg8h!iX?ZIKFoo(QWt3%M)34h{*J7{3-(tLYBSkna2O6Joc zt@KH;zAPYR3jRwvpevyqlpQ2ulU72=z3<;Ig;cv-EySi*5=~Dxm979svc43i#XV%$ zjB>5hSX|-~q-`Zg|JbMDz)D5rvh)7EZ_E{=+)wC(Z_R!GUhYPZ@PL-vF{p&9%Ry3| zmohE&rGLO8fh;Fi0;ahsVi1|?Mv^9}>^M4EAIY1B#Q;=$2-o~(hw1sCp}d8n;ebt` z$GRQAe=ldNmXgQ>G0tSFVv4m@j9V5c35@H!JiYjARnBP|7G~?u6J|sBh5-(s4bn<2 zNO@USa9_YXX`FP(^GPBNI6@WZx+dg9IE?gbRDbjAm^_X`_M4NQ38g*hmqxHr1BPd> z!Yj_inpmbEP+CgF6dNP287gJOzmLSbXWx)id;&EPd5voKJF~xE(9D2SRfbhD@%Qg- zFsMbFugkN|VpDmqV^8X8h%ii1sSEQ4be@4fA7~a~KQL|t=5X)|vZy5045U>uMRzE4 z`+rsWWJmc%;w;5xKnyKv$S=X`A4Ag-%6D(xB|}a0M36)=0jjK!0qcib#`C*w7p#9q%$Tf zAjU?@`%+faz6;;y@{_LOIbN(*;8{dqus7rfGKR@4c@9=ddLRh;d0ruyrC{ueIXH%D zr&tMRd45%u5RF0gCbQ(N;lC2>y?(HN7v9>jdT+`Jq9=XIn8gw?%h?j|iMKYz*L*rg zM+Sed&f;lKhqr?+|pJsb4bHlygL(j!b_V zi464=&?o;`R$ohYT1;JV$-stZwnDJgF-x8$aLjnr#C}&E$?%wVp9V``me@PubM|2n zW+_pevsN2Kxot&xb|MGlP$H(}ENj~!@yu4+hM8(4u!e2ps&$vV;9^{& zWdtjA0N}F_^Ea@BfI(NAPKWMs@b}!cZ?Vm`v&(4n?$FuSwh$uDhM_5U zcVeu8|HW}gv@Wxyy61m?92Rcq{M+xq-0(uS)DT7Bp~{4S=xqmnaZuS%h=Ni@gI{|U zp~GUc8dGK(IPG$dE;d&EzcZQkV^UWCSYR4dfneaj*CTxbmk~fL<#GZ81ny~&Y*I** z`D#@btL&z%Gn^ht00E4vVJBkpU7ZG7Xnb|*P>Obujo zxsJVH$9Rr$U3z7d?*dPy*R7*sR?2a(%4w(BJE$TjwYH-vCzKP!b(5ou!**|vjY$4O z#0jAH{DcOLzCw4BcVgf<26#!`C>g`4hO6}$85XV!Sp{~u9BnD)bd)(}D9FzWDN5tz zXIeY6k2Gf*`4WG0!lKrmI=R<{$m#I7dh=&T*`SrBOxCOcXOo_ca!Pfmn6l#OjDgCE%+3E0^RGW( znKl`f>34&#GUZG7BQH$IP3CXJ>`3X)W0mm!Xl>7cbk2X6cYtCpj^&F~%rWM)lE6@h zgvB)1D=-8Nk6vPEIP@37v3tasu^k zD-zb14+rw7_3Z$qnhWMmLb2e35CM~AqLL0Kp;?WGL-1N03lxmgJi%0QO7waOECXin zhbJHRTJRm>rjB@~p3aZ^R9g_D;gCIG4ifF~gYi5UixP6u0KqS($oQu_RyzG1B~w6yqKAVZh#am{ z=M>Vz%gj8qmk_Ux4^}A$-APELK-^Um(EXw%x7nxMw>c9tzWjC3XNb5My~xL=6P7T}&^i z)?L1%Br%C>=*W^>l(~x!fYbto{|>`o(NavdRO&nc1e{RGn;c((DtfA4_qWfVzJ7o8 z=H21*XQLmU|9zzXVJ#`bJBWi^6}jN;&ll4Pf8l`->S2!3d2u=7y``HYTu~?=GnGyz z;uX4pJW>-PP$?}WKDW9Rxz2F7j&GpGfaT|7JlBHqNDvrFCHZtAtzjV3(6fV*O@_%; zAVdyp8cp-f9CC6=P!FDoS3%s$j!A!Vzh10!nRL7Ie3UZ5D4ym}i@pzeq0keRZ?d2s z6$Y8@KkFf2H1i6^K5O&|#JVWVFOZ@efmKECA!1kYDtR0&acuH9D@{GCBn{b~n>-UomR(|U+$XvHURg1w--^118mhY?geP25;Jz=g0N)dl<2L5JfvJl49<}T%9?S68ch+O)5&kSY7A*!B_YSitv zjmC_nA?~o(7Q8TAUrF+Ftpd00&6T(@z>T0#Fb?oaY)uid5p19|C!@?ziVD6Z0l|`Y zRht~8%0^l45`BA$p_jy&qEk2u_&7Ky<#*!^(6hJXUeXU7j3pmCa|(aS&9O<{d|8yn z&+hJu-`j2ppV&QTtvxep-62K6-?O*3C$^W=Z88-Qz+!MZ9_;(~8H1ND0_h8n58aMzPXfLzDmJcN3=WJKtWpYlHm@# z=s1RPv*6W6U^<~E>l;XSGg(k2>J6k7Mco0a5+QHcPQdzvLpa+5YD#CG=mQiEG1P|D z8j;c5p00jX?<5BwO2LvYdW)qt!Ch@N$pCs-Q{#xnd^p)t5O{y4DE>!Im(-+naXx`q zpHRGyN9mbj)moVO-CJF^HY-s0s~ZJ?RUOUsP?!{YSR?rwA$^`iZ&$FH6| zt`Yx@EJR`uZU}$dr$JK8%C*h_O(wC`SjM;9C!ai&futnsaK#-;;bGHrEE$_p03oUR zl*|b7F-{?s^3WTl?ICD;S*BLs>!)3*Q~((ObO9kHf}&A$44hsT>9jq8$qZ+e)!ctkq7uiwjsxtT_q^ytkf9IG@zo)) zHL`~S)j0&zjEL@l6Q`b|5SI2*Owgx)`peLU=3r4v)aN@a;PV+=lM$(X8am7dE#(`s z$0*Y|smIXZp!6=z!)^q(Q2Hb2`(jZ$b2T>C|+Lb`>tzK0y zZ}EQ@%HwKe#a;6Ziz+y!9r9*Yz>+3ZqcOGWHUlL zP{hDe6-5Jxac-MEOqFZ{NS^QX8s0;^B zbo?Yxf=f`8Eb~fcTOnr!NsjBN73!K?c$s{s0FmNHUDGq2VS*D<)jz^P0FtU*`C z@JW(HtR0bp^Fo6!V_#Ccs+i^TbvafXAyNa-B3XZ# zAyVo4ry2vK<|Gf4nxUcu5%1qelo{{eqxDTNtXYw%{0=BASg<8HllVcvC->sp~QcvY5El>s#>bh{x4 zj}D2^g0B-i4YHu(BU~vjyfb)}$g+Q<91DnF2j>EmHPAcHxwJzxgMLoK%~nR5--V6F z2%kLS!(q5u8;atI3_q%0!;36c+vnvf8f9rVZ7Wgf*g;R$YJRB(r!L zcek#k2~|gGD?)X?BQF6`C<)S^O3&Ys(i3F~I(F$`EX4<`4k%8piw91B!|m!u^@Wmm zSSrtPn`CMt0k~=t34ji{*~*Eb4t6Uw2j0p}6#|pv-^*$QQ5Yi`0wm)X0PZ6_4-E0B zbVNHq{6>A;*{KO3cNIBK61aao+AOQxV5cG7o3ZKPyDem=k=ZY`QFhL@ws{i~nAz5j zS@Raab|NY0_)LQ#dK8w&55sL18%3YL`RMj$pHt*DZ$T)8p>| z!&KhoI-_Ju3vo+Tk=_jz!9fk5j0BTNnz?!Mu(HrdM&)!;--O1uG@5@pkKe5Lm6E7I zo;E{;e^n+J!DVr@cP#Uiiyd0)uKFXq9fr@5tq^RWiS}v%B*34#sbd2GIbd)9A>)B8 z!f*t=kK2)lp_OA{=Wnd?R;K^n5#0Abi#b*fO?U7oauP>BZ!BtLWD>}G3_mM3a$svQ zdpx|gke~v>*VXQsJ1oPw=Y=W-ZbAb6j+=pq2rVx_mRphME*qh%k8 ztE$83yH~&5F3tT|ASZ=YNcVafIy);r%A17dtHr!W|DrpFLw%j>+Nn8qoo3>sf0m2* zWSXWtfhU`}yiR}aZKBARjy_d#ejeD>&hR=x8TccIJ{)TT@vN-INdEZ*we#dee9c#w zcwM~&OHENAYZ8Rf=LTI!Q1f7mIRbfO(rq-aBgxI+= zv61e5_*q_DJ|FKUu!NE_Bug(C9nSpE^^5mA+{zv(Hcan>2^iujG2Jbo8PN8q?Fh`! z+ZA+JMRtEPv?)UR#uB`KtNOk1+W*p9BPZ%@i6~ts`pj#9hFkce_aIGI4mvO$f8-5{!yAotYDMSc}ksN?Z z&S>v@6$BvFv^f~70KAImBO435t^>$?I(Q#`P2PXo`aDg}+r2$j^*Ula=EcW#2TEUb zWO1y$;~;HIt+P8KBBHpKvN;1IFnzGy?2(l*LexoM=yfPo{edEh+`QjkpPXW%_ z?2Lad?92Ew{4hGXqDhDeekkWce!!J?@MDS-fS>B&`e;=OpO02piGqni*12R zs+!4_WbmfNSw6nv;k7b)0wxNwFJ?r zmJVy?%lNb#Ntod6r@2+Uf2y|FY-B2h48wmhbJXeCkR^w#j$8I={RF&{V|d%CQY{nK z#aMyJ{t!@~%12RqV6l=$9_ZHKyF@n|h2S|Q+$4#_+N&eysvXJ&2}< zGVe7aENvGTHkkPOm({1F^RqK4+|ExarqA7t!28*q+9xX}yF&BV&w4SR68lJZBME=( zl#oj|-`+a-?KpM|K1o_G0~c!vIo`$rmJi2&*FWm*NGm?-I5cuC=iscrI4kA_SW|KB zkKerLCix_%nro7V$k=HL)q-gmnaHBM+dZYf0-cBH8A>Dmm9N)=%)pYm$*!5adm_73 zz)@2V4u@tJwqvYNT@+uqE7aop^(k>aXOuHA$S zONG@Xl~)?ofn9Z-*+W8~P}~gs&IAO(bSl;mcz^DOB`(guhyraHN{63R$Uc9FlOtwH zLDj1`44L!!Z@4A1Vtu}t$nc-jWV|XsJ7P|8Y87?;IMSAK0A@Zl(M%sJYUbR6y){$% zAY@$J`~k`^pmjbHZMxRM$R0?!cU>gdb+w(>BC;V4>n50NR%oFUD1hq38V>N-0pbZ$ zym5m_g%lJra)Z@XGy<3XVN!o>W;RLOmcxn8-D}nI`LT)6oV@8Z)!3<_R5ai zze*NMS)Q{>CL2|ahgg2}pJo1Fyln4lcuuhax}9*%+XQ9>fCL{>%z@5e5MrL3U!8zi z58pU)BU6QtX>U2LkZD)QpACBGSQ*NDg82z#DiGJNwPUBR>Z;V^T`Mb*BpuQPk7Y)H zlxi~9K{&wy_c2oilZbzZzfuaM31x5Cl+`);FB@--LGsO>pOgra=!*^rOu7o9*h?=` z4^}jh^GLgiox(^1fdZ@A!w{Pq{Sx!FwliC4$)kf3$sjr6esIUiuj%l-fK6S2H|P?M zTFeKX5~cejoExH>!6MQaS^kwlkKlnUuy#nf=~z~x>!z@ZL$rTWS_M6ggPkGal9kg6 zT=#T3NzY*?I)UwTT1BVBa$I$S2oX(f6$M$F>gt!wZ55qSB*|6OF}va9d4?+!F%AT0 zvoaY}dAcHHg@>0>858+$CT9ryz;BTGwSnO^>GfRO|1Gn+f^7uZaK}pPmCy@{coa|Y zRn=wlviE)`oMC^k*E+-C8%<|qrHMNxO;s@h+B0%#kM@#g$p)A9_{uRIXQ4^o)=SlVtCNU8Lg5eHY6SK zVBUiVQ)WV;Y;7uk8+t^mcmZNjjpw}RnOu~`CDnQZbK6opK_dzfq9SGV(jY6sqE_yp zGkvAsc3^%o`G`qXfY^DH9hI{=Dmjmulwx8~Z%6D6WX+K5%srb)2}KZzjj-ikXAh(K zJ==D4!}@>P_OsHGwCSWmx|HAaHhq{ca8ozsOFhleJsC`IA5cfGcot9^od6-%%!u-* zG$djoi2ec{xY2d~R}|($?}S`2&)Wcn64hO!e_dC8zwVPD_uo?jgoF*I4S?dKxP^%# z#1s&(ipCjApbyY6k?_B(RI6`qLA&r9&;(eC-64Ok3#H(a#(MdvK&C`icCQNflG(mJ z+9U5)6wt=~#^!IN{Uzuwq_|T^TMH_3*P3CC&mer-aZEMrI&gh8OBN#s`OA>ZA*{ov z?RC!%grNtLDzKr$8M?R{r>l_q*_NVlp4z8`O+Id7v_g$YJ=At9d17};HaW6X$oBHD zuUUUnRKv5h`Azr|fIsdL&0Yl)9XM3JIg?1QPA@EQ^lv2a-P%B^}a#?BnQ$ z#^rM4=N;OJpf?y7YY%NVE|Xh7?}C|>`*~MvNyDFaXiMzJf>bwVh6g@`>g}YB&?0fAQ?3~S+B4RmfL+ePNM%?4f@}z6a%xKD z0GgGcoeO+zSH;v+6shnk}pIU!p0@b7# z@K0HCWokpR;ZDmjddpL#FG@HkJz|bC99%l=dyv6IDx?9?iVAiANe7en%9&b} z-4NPoewKozZA`wC{Pw^M%CS}A1LB~ux|as=8FM>RpW9M4v;KP{vnPqz)czTB5i+x4ibaSft@HO zvV@fj3?nx@c(Q=!d$mFDq2H=J*+`wI+!YuGUKI>RvPpi77vnGAQ`>#}Lvck2V0_%O z2BCl}RCSP6J7ZmHNKna-w4-aDN2 zd$hxptK_{R)SdKwa^HW0^j?y%J@FN8**IUWu`oPd5M?Qo#+vnlm2=2~?>3}15gbkC z7zygi?1IT5cvfK_Tg5bG0TtP>ZMcjF%vR8ayGGQEYRO@hoj-X$;IbyV;IfdBQi8|0 zfGUE!DN}nR$wR`n+RPlNbTVb6I~(+}HqI#7JH0k)m|| zhB*YxCkzg`>4vx5-!H!*zd3H{P0ef9&H1;+Uf%+nOj(!q0@205$WAHGCBtVs)Y!vz z2MgQ)6BcrBn?ag`O;CHFDgV&jRUTF!#j)VlKrsLXSI0_+YY}Uo@8CflnKf{)&>at! zbkNtcf;24{VrqYUWWzP$+l9c8!Q=VIB{5F4{gdDiWimJJIw!VBAKO_W{nI|=}Yq5XHQUp1PHl33OAt;SNVS%*L z4@CtPqc?h@Uv#lROL#G^IRW&RD=TN66jDbHqFca4ITXQk9r#R-9PDg|ylep$XxB8d z*Zg?7jHynzv(13A;%+8)4MoCj&zK9MO*1wEteS3c{2&;a;3vL5c0DV!?xQS7$~6__ z5&DAyq0@g=NqP<4)2^U6n|Yz=nC4{$$Qgw$}GXL!bO+;PzNFT> zI6rvCd~`k6jeaOW*ap;Df07Mt5T%l=Ky(%GPEycI(5u~bikgq2Mn>eBH1 z@vDJFp3fafaSkd4?Jy25l)R20CX59QTn#Ly_2aeO!!_x=R*EFI~Q0UpSk9>^@VR>nmv|`*yG_1`81thSxx6(ldph5*sG_a z)5Y0YvC4$FidE@AvC@J^AQkGrRs(OzVjcf_=c7_+HFFBPUp?u`)A>44i5EdYft-J^ z6NN-A;3(`P-SQ=5CtBuL8gIZgRA3^W`fj%o98(N5w$)qSy*_6XK#0$CekWW~6K6Mh z^1tisynSyv#3oV4at2d$+R<8t<$}q_ z*aCd1Q^Rp5&@lMOr0_xmjQH4uwEr8}T(mq4p^UOUe zlckTAm+FJ#Q8ezcIVWP8e&R`8E2ht~#_+Q1(A$sv4R}?btc|_`*L<=@!!dt@@hP+r z2?E#9_h)VV&G{rLRT&m^F!~vlONju6=L;(j2Rb2Jx$YRRc{Q=rPCQw5<;si>*xILn z5x{jh6WCLuEE5}P@QPiKfE`GdK{2X($t;h8PG&JJepUPglByFjWk-TIfh3gtj7IO^3EPki@9rQF|4LScWR8!w(=bSZe#%lI&E1yk9V{T=X|&|09`cr(e}?|aE3 zQxF3Ft~*#sq1{1)`x*2gm%Z#EVi4q@{bM)M0J{!{;p!1J0FMUUWU$8+?#AyFid3Nv zuxU2?$Bs<};!Z`!2mAVWmFZgMyB|-=3s#cE^h&$t?tlU+IP3@$=}3RtY9+&NzDbV9 zusqAI-!V9Hw|;<72?aikp?7Tuync`V);&Uwg}wvm)5lN0f6huE4B0ODGW8GdUX8wc z`}*CRgQ5G}SnUX&Y$=fh2I3m#+^Gh7KNwhU3j6V>(dM z5SX{G|9&tuDC}>dV1j>_)v(_UJTL9;r!KT*==N%(s?0I}w-~*S1onmes?V027*%(D zYipVY9cLTbyI!#k%`KPLNZSS%$G4j~3#;|~2$kANu>Rk@dFT2$>GT|vVq9Wk1I1&4 zt6?Qskf6t*glN`NOv_n0&$XA9Ro)5HuUv}CGw4ndsF^W| z?@qZ6X3+!E3rT-cl$ge}&dkmd5}KF-Ka4-`CFM9m8e%-42z1TV5vIU%GgMd8gUgfK zJ91(90Q4bff1-@tW(nD%?dGg3&)co1*d{6~PAtcvB#}WEQV~zZ%r&h7Sg{|m9A|E) z9PxNQ%{M0h6U!`i5p2KR_vsvVtMmDh)=ghny`h_E45nG96A&$^eeoWzYK5*z$qW&NCYe#r z2}WQ)a_WDBX~*fc&rGy-N05(N8B(@ArfV(fwn~1J&i;T9H%y~D>Zq>hYFbwqrFvXP&F{2K z6%TG_OzCh!Y(J} z{&K8)%G*tt^xKLN%(aZ;)CPjFqM!)7F{I*7#adLO0*PcJMwg<^K>@H_R65h51>OjM zgZ_V3Je!bs^Lb)l=NPde1UJOTUN0*(%gtrIT*x8{gv_!YdMIC1hI&)2+shSNQ`Pkf z8hHbH&4{}a_BR#yZw-)^5h(iYo?E7fAy!=fBt{*+0)s}XYXeEm!bShK_cpF5Fe3*7eEn` z*ke3R5byngcNd4_uWj`zawMcPpb48zC289nMT@~Pd!W!xT){^Kn2iw{S^1ihxeYcjhUUbn_=o|CNv8X4yh*BxJ>I72M>MnmL z%>y%EiI(@5eBC$2<3Fx=baLJ7sh#|HnAW})1JPJ_M2aROL-){bI$&)(O%N0xe~U$x z{nbswfT&W8F<5GVIgHMoB_pFh;o`0AQ78?tqj-=x=&3;(okI1AHn4rl?s z161zEvq{iwqUZBsT-PLxd+q?*{sP-ISR|Mfd?cLX zEV88lkQtyo{!g@fNTJ7B!YX~b8v9fbBY+~PA$6Bu$g{}-?gYDChF_ed=;VJ={pFUs z0g1f#DRbaI!^JuST=u%TR3zj1f>Hl zmx^CA!IIBsq04#kSM^d6X4b7$))y7*cH}2%*zv-2^&ye3^ajEIlxDS!en9$%KrRxW z88_gBauCnokUtMsSM)knFnfP?H{`&}C4z`^*G3IM{Z)BMKhkv4tx9IiTCpQ@dSlbp zzF9k6jr?(@!#$5f{V|vf8{G2<23nDy)1LyEcc61LVm`Z|wG)QgaIjzPY;p?O>bzAu zg5<>KZsP%dztu>ccc&hkkbrkGHC+N-ahW7vjyqN3_T)o`nW`PXb=i2=u8DVBE`FO*@nHzXXl}$9x~VtkgMp#A zqH|5S&L)L$MZLlFQ4QiN1|h-C3iwB2hcw9I z7NNMs7?LyKK#PxOr1pP9nGhFsv!I2v%ktP zF&*=mBO9IHnu$&TqBmbJTFq6+ zmv-b>5fNGUy~-}HdO_^)37yGQvjT#S-d39}x=x}Ew%o_4*vx;~ylJWwN(;E?w}P%{ z3K}~UKxt|+r4znPRI{^iB;D(LCDkrm>>#6Z@tHL#X$#)hxN<i z!#9HaWzY>f(WLTsE-KouixlsxnDa%hR#Yq*(F>eHnER{w?CyA-j;6jJpf zeb7_AAgI4u!-;<`7rJo6JU!PufFSJZ2Af?SDgz9#SJnS*7K$Y*_5r2|J3xL=5^pgw zLqy!2;PAM46uSRs*t7m4-O&D_fgB@lvb<_9hkfAS)rN9O@XNpsLXAQL6;@PT*sfO% z`Yd40pL7wSS%`kDz;Fx4Z`3>MH^fv!Yz(%l5zSv9R_K2e_S40u6~{3kqhZi~z*g}? z9pfM{M+8Zf@c_MXX60pYnqz-VeDSd-!$H6l7hBUw)ZMkn3^v6JcOS>LWA8!3;punm zZna%(wyV`rQT1h2Hv|72irDUZ(+ckE5#M;dS`TQH?u6T8m)-0K@MFK`WSf^tUpW5} z)Yzflarl3n^IJKbr?Xc8bs2IoG0$X)aTUy7z#<|I$xIC^hmk0rn1!fxv+&AVJEE^vl7rtN<>)du2gu>BY6ECAv2bfHmCD>k{% z2RomO2={#6R}_k=MeTM`Va+_G(qJ74UVcbS;>Z@HB#Q>KaUnz+`<#rsfD2U2 z6XOWTa!}b4JQTqd?*5Pq9~?ZVI|Vye!WP)*>5A}Av3H3Li>y93%w4Z5;jHV9gxp@D zt&4w$#lyDDIbfpFk+{A3${8vNh(7xqet!bL8V3kUH2HMs37K|ug?vrF&`zh7U?ICW zh-n-&^m~vhMq)aA>ZGm`FW17mrLo|3gUwe@Uk$#$Tr_{f3DmEm;iWbP67N1hPCEhYGWw!YYnH36 zWDSzG$JDV*Dp0z&sEzHpOS69`+DIsy{$(uuOEXMr^hZ4zf#fDb;x>)7%=L}Xe;B;ePM!5_NF{&$0dcx8E1>lqMs~(O!iPxix%@iT`llNfQLCl7H^fQ0* zwwN4=ILraM8A+3~TN1k-@@HD4tXG%6u^?SV9c`l+^))lUa))q5@qvU<>esmz1JzA%qq5f5Xf z5|kZ8SUaN!V?Wa6u5n;}sIjRa!?%BbcqmvvFI$!tGX`Sgg3YtIe*xePmbd1Yc|W?C zLM+=Q*p#G*)D#}{-$DJs4dkJjggsp$Y<9*}%dO+XGfN5zi4?Y3c-D{wRjJ704Cb8}~gE54kc1p9{XUw;TB_8yJoTw?QxJ6sk`})4M4N z5nVSv$la|(RNkSJ_?Te#;=>I4FH(v}UPoQi* zT$kaXb7ktsq(WWBlR%ub)6##vT>VMuzTJ+Blrng~Tvno&PC~EkUazdr-*^mygxzYn zymPm~otpfeoD5pZFcFLV3NKI1n%j?;%Mm63%1+h;%TpiKD!_gCPSI-q|Tado$-IoAl3)U~`_u`~SRmh60RpjXY=C8+L*Dxmui%^g}k!(LjbcZoE_+?Bw4@Ep}{?8Sq2@t4jl z!%JXAxegaq5KFWjeGvD%h5HBBvYuVn%Em2oOVF87`x)|rfQ3!ZUr2&JZK@Ya4|Y~) zidiSq@r5#U2Ks;V+`6UU&IuxLI>H#7@HccMyPdaZq#q;C0!EM_n8jW7uowa(Uj?p$ZYQ5(#=7PqRJ1#P!97 z>-pBJrmZ!I<9mYifmUyklX*zaZjGj?J>*q?yQ$YV^=1K5XnVO*#yyrxAUd8>7U~PN z4eM&dW%hsCGrs+pil^-vvxDKO&ZoPciL^G|^@!iQ)EVCsp6#4Zdv8y6)&|WQXI%`_ z&~trfUTfj8-sP*F2KCLJr4>UC%UD_oPF?P-){21alnJ7A4XPCQPMM?8aEeHwV40!V zZ~`B-r%VyAeUx|nAW^TZGz%(Ef`kkS&gx7e!_77fz`JUddMLI5x>883iu(QFmFB;~Z$EEu=Nr zyS0C;L#^R0ygH_tNJ&bsFXV8jQ^ zp=?1l4dxOb>dSZ-;;Ipl4h7k_80qb=p{(^>@2VnQPtrm&VEm#IKh4K7TN3M2mlohj|Y^&gbJkYq{#MuOWp>eW*xjJ~uiI z_dbQDnFd23veVEJ7=zjx@ZfoKU2W>BQB)@DBuV>*Qhh)*#;A&7>#Ss%+$6lcSjNwH z7xViFW7nQw4JUV5fkJ1}22IJOO@o}fdeQ%hYGD^5-6q-mKpe=f0O6EyUTZIRhTVTT zDy>NfMqPwsrZA(Ef{-6EJLfzR6MVWL2=~ny9SAx#)Ec&!nhJn}R&jAd(U{9_DK5l3 z=@@+05X&<5Y>;09C~$8^0Pb&Xb}W`#&=-8S`RSoo6mZYKZFJY*nlWJ44v7*+KetCs zW_%dFkq1m;obwIbOX75CxbKLHowhu6K=V=4qgdY%yAqCj;bb}#`eck&(_a59z3zT2 iMzp)13WzuU$gjTreEa$K^X=ys`SWkOCEktzXbS+Zzcr8m delta 42670 zcmV(zK<2;LsR`$a2C$yqfBSOdMv^bQe>1}GfMv(*fNl^iwJ+nKO~{yGnE;(Nkl_oyNXw!YZV$;&)Z5et&Gq3lebv)8bY=Sa~UqY?BiI+Pz(@MVO z4dHK;j@DlDxQjKde|EdWY%i+=gxa_k2QF%j)$c>&tEClb+EfjLie^KjKX5`9KbiFI zxRRSNAj7#|986-Se!eUhPHe9YPRO!{P1g)Wo7ppv=Ib@!OXymIhH8N-X(73ZYl(nR zrm44Ljph#gdq&P#OFI8REL%69E)FS_T`tb4$vMbq-KaJ?i$3>52kI{(Qc~Mp(swIc zN+EAA5wojgS^GN41RSC;;{g?vGk;atl)c)2!5Mf@q3*V_oYj-0NmQ`D@LLh{e6%=! zX*=z6F9OW(S&JPV*KC(}`D>*8bB`&*EBJTWiCrfYTv=@4-NZ7)Q7F6ZlM08GJuz|r z9G3KMjn{rcA*bK+$AQaT{NnYWhkjvVaX(&OWx1~KWBvC*w;K9K2FA>6@ni0OZDL1% z)IXDc`{)Q5T?34k98&U1i79>Lff$bKOBw$j9l|f8lyyf-E``!rH$0ZtmzhZd8h?Gs zca$q?|8{%WJhcjCiEXQO_sV)JLqKi^r{;EC=;`Hpo9Q)#K9XZ|O);fstLvp%TK{HC z%LOOP>gC78chQkPwk19x@NM;ZTiLaL?gx=7bam^vu7ZZL7BL`zzu}_ZKIAjHU> z5LZzycLj4=DS5er6L4&L_u!{|!Pub58;Zisp|vDVW89qf3e?REt=txQ6pX2HA0%L# zt{${zSTW2EIM(rg*PBvobYrw=bo6gMINtDte7HHRs+C$VRRaQATVDNI{an1j+U$CP}Whmgosm4Gatq3 zj|=e>Zw4U;{2*nSozW;twG-TG;q^^T`%6KPV}>?%d@xD%(V<~$b@rN+?O_Qa3fF~f zy>tYi4q-=tPAFiri$5eV4P4(hhOQYz|`Qewe#vN>G3fd(0~8thlm;noPM+3g?c`YR|s@X&agr_uHtk0W2aX52LN zT&dBxgDGQtgPkM9_#i*5$hy?1pj`&N#Bu#TfA#j?38tgs*CkG;;Qtk95OE(vr2Fr; zDg)<-kxfWYIR@$zzo-?prK;G`-Yu0BxSM_K>2raxf#K$LkPi5NX%T}hj_fj@6B9+i z>1$v#M_<2v`dTh-XV<}6uOr-iiZ$D=AX*CiOfa`n9>I}02firuWwF{EvSe)G`5!@& zh?4ViT5cuFQ(hum6#}G8)6^Gu>Vrm3!+EfU&=zgec|gn8eRt`ht_+!E4|$UJq9>rsEXrje zrzL+UR;x~zvRX%W;xD^0EQ|-cHH2f&cZ;QRM2r6W|NLKnMGmup0mkD&hzRJkqv-YO zy0Lno16WLgyEeEy9EM-v(X91qxx)4Fte2Prr23_YKHjFXHh3kH-@_K2yg5uj-N^}G zS6)pJi9cBD1byrq2Ij2)#@wCih=nfxg!F!n`c1Xrbg)38jF10R|NV1+uff1d3Wt)q z_27X)nmb~D;IR8^?bult4tcK7M~pL;FD8$c?Fvs9cZRr@yzf3YMqp0iT=;MN*XToI zWW==o6+!T=r-R0Y&LaXJHngLC1dffSPMWp^YWjM&iiLS+pG~6Cq)eaJuZn;vb_<~kQ>9nVib_bFxt%7_CPjiLFmWN66z5oT6@lxA zoSXa_aPC^f%UY^Z?J7AfPg#5K3nktVwBWR8q6Yrki?uWw;3kH&KIrVKu#4tm z@*5lvnTaEp>!!~>Sz+6R*jt|6L?yQurSf+YGV?ZnqY*{1f_Tn?9?V=v@r6kyhOC02nMN~=2aT5OoZ6w(wbn zDReB9s{}(8CP_D~wna)Jhb$Vrv7-VvH;EMFea^&|!p5p-DXI9-*x^ep)igsJ*Q~97 zxdyGSOb!r+Pb9Y*Vvt*iNDJSq^UmL1?rSprG>%WZkmFhE0gO5}zA$L6&lpKMeqCk) zH@Ry|TY-Q4@k&~>fvKahAaGK&k&-#vYGa|~lORdV=QkASAn5YhULE;v1q`Qx@E^%@ z%I#g8hPOB2{Sv0UecFy#N@6&b1R6wtSc;9ahrZst>1@K`o55~oR%c^b#9IUvW$=N8 zqC5+SKYSFuEv6Ssacs<^*9ExkCKz0{K)>9oi6ZiZsz!j6UVU(8uE6M}Oj^-dxvA7D zZ<+fRj&Izt{;TQq#JNs4Q-amhm!5Wg^J1%kx!%Dt+rxq!I~w+{=*K#Yn2{j%nU(NDUCKkF6Q57CnD z1r)_@T!SM7e*GIY1-wcRQaUY)Wh7C=AjOfqNfAoNe5VsX)j`B*m=ahqa`WT3xrmV9 z{+fpBT@ofGR+R*^W=AaLgWs806Jw7#h|vN66bdcXPYKGM{1`Omn4Q3X!io)jO>|BD z4>vqQ5o`rJ!BdX%@X8QV4YW^dx~6qBn?(jD(HUJg!C8y`4>j_t9M-xRw4WXt$T(tw zZwj)>kps~q@i=ZG0P%<>A!~2H%&?4lEaUf&!L=p0mBp5MmtnKOp+OofTZGflmCLF zA<1LgYoV8rX$!USjI8T1>!aeJfVtr@DDRr~RbybsZzoZ-6?3 zW7Qn4KzHS?Gdip{Ta81~*1eX_$hUppm!1UHfBF5tzt4X){(JR*;C#OL1R~ZX9z5lD zc^}~SPWbO1{ps+NhYtSx$Dcm@)9?82zZv}ZJAr@iiR1buEUe*Xw_FOwiR(*e;CcSy zD?=0Ldt7g`|qC|J>TT(3m6Kch8Vi7&?hYj=QIo~6oj~$%0zFn!neisYXE7hwok+`*p)7?vMJ1GhGAl7d5w|%%@2>^h1UsD$!gBy z_3rt4_Xs`U?PAxf&81L3&)2&Ya3<_toI;XA#Giot8TrhAYnf4-LcI_(v_|k2j~>C3 zV6DX>EtVMYNnKP`pVC9@UAS`1>#B5-FHZC9;^uTyYJ`4*ynYF9zQPwrqAM?DPZIYf zuJAWNE$I!|k6MFsq?Y${2=eXqdVj7LFa?`G7fc!Uh56Mu(3-(WCon5E!woG3xzEIs ztab~+P*Qt;?T=M-die02-dZ$Qh1_CN>BToRKp78hW5)(^^}t{PX@$Gj!(80cX#qEh zI?+ql%4J1i<8yKgY^bIcFED0MPC#6#m)H5`yb7V6TZ=i9gEj*8l2~9oVg_VFt~z87 z?PVAwCyGyM%yIf67-(cu3BiDVzm*&3Z%d!27==793;g^ zdc*hBgHLcChF zR!m`k6*Fdcv$4UZfk(&tw~RXt?xcC|H*zGJ{ze<82Ty1`i*gH%V_KRCdHXljT6+=R zUGwf{A=;9~*wVlLK257dK*}cKwvayYw?go>yKi8By599~*EDNpf1&0D#!rg_;ylPC ztCyEK;+K&}AOZhED3O6^J#ut;=By?}tZZI?EYG(W@?0}X;3cF4{Lr~6J&1Sq*oE`Q zCjW=vf%>oDKi+G+cl0dZ{h%B*&6jYLi|x3BkdKH?2X#cEI^Aas=}cG0KUqEa`Ev!w zE}liQSBqT*?yM|HE0)3#4`e2DexWvb6_{R%ID(2^a4d*3KGV^AdPlk ziZ`q4WwA;99oE2ikM(#=QrNUv3>Q0t1{$p=jZgB>*pA}ELci8VF>DdxPlTmnw$5#4 ztZg*zY6zHq16N!FLbm<+kdk5;*TIHWO;{n4-u z60gRA|Hx>8|8lk4j*Xc&JCFpJ_3awpx)txHFN$#Y?kLxOn8kz^WujLg!xl)*paO`0 zka*{e!9;@#?!ut9)3k-_4fPA{r_Sr*y%s^&ZF(~T z97JD?^(2^o%L#!;p31bh4nW;gqWyHWL(~P3$g6x?R%bU{N4qr5Y*j-~jrFOLB#5c( z*qy$4w9ZxqSs}3NLillin6E$h zG?F^Ag~t;cWfoIdF$oPEo#zS8lQbDbz!xYtN4|k9TzV6h3kjtv_Tp(Fb|ZRJI<)Go zhhc{@7eoXm^)jEMH|rLy$>?~iuMlu7ZFDV1XqAITIY(VDJ#&|1EIE^Ev;FwG6VWDc zvbmGVm+%j<_}Q-vfK6ov?|eH6{hYI}_TXS;Ju4652{ zpl)e0`2a=0apE(8yhXm|#_^I}C> zM&t*UnkRrgr@+UoSxujP|Ki2S+kp+TeI^!dvB9M&8tv88S14{>$=>+d7-hLnEB5`a zk#Sqd5-QeYHO;1mAsd5j(2XrNskl{^>-wkw(TLv9izV}QWkgheECeLHc#xLcViB7L z_aC8t1+B@1^eRT^RETdghhV6fIp5}Tw=;d@a7O(o+o`2Zx=t1k8|K2T%D2KiR;^xk zxs6GeYoor}qMd89YXr!|y`Uy&@!jQ`xd(A?W13nqH5vD$*L*g#(E{OWyMn_Y_Wh7L zIlOc_|GJ-i#+jpkfnNi5Ph*TtFDip)J=vix+`o^PdDzf-s!}i!TfF>-6oSiQx83A( zgOV9Bq?m20^u!24peytykojtBUliq9&}$GoOh-6X1jCY){w_HhaS&IZ1?|Pyiv;-r z<7Q@mK@L&(L4Ej3_X6l1oju}s5M2?+#`zNSGa`r%n9%WmIzcsmh*YB0*%`b+e(Mlh ztgc`J$i;^4I`l==>N#YEyBw!^|g|^vPj!xHgA=L&0x&tOjz(a{~mGRHG`X= z96?~<;6?x?puwa4#$nvc63_07(u%wx`% z#4>V=X=Q>J7*}dqYvbU$Q%iNsui0AeAJ>1{MO>j?$XeqDp})B@tO@2>;z`k3RTx{p z+%cyeI&R;#hGafA{$J}Ia@Z7T z?XIdtSR`-?;-8xtrWlHOuFZF^m8ly&4foo=@v>r)YU9+Z+R6l!EzS|$Zn!GU!#4Bc zP`o=Y$AhUc0awU_yVb9Yx%L<}B*-&50O{3#wgX@2%g`#Vd91waknH}l*er|ram;)+ z8B78)V$(JFof)Y2_dbv;%pQ}&L7U*^TPSeUsM){jxS0+AEe)*iD(CM(vvXEQBG%8b z*ZVVtPNsh?w>@-m=rZ6mu*SEt^;T|Y1ar ziw_1yE%ejF5`Y#GJ8crWkf1>2Vb9l9zO)=NjaD1PoW7XE@S;Zl+$Cg)0z)mNnc5lMouV7FM7}Idg%Y zickOJZT9Nu@i$MOWG}ya^8JgaZ?cz1uU>uq?Q=OR>dZG&uRE{mWZ!lsMbyJFg`4|! z@ebafo*jPb73zRKX{?7-s=_Lzr|h9>*(-eeHNAw<_`nS#59F_pb#%6WLSVE)zbViw z6zMIak!t)t2lCR3#x~HiBo={rJ!f9cDGpa#WiaTbHD+BAdu}YBi>;7BjZ8dqligq39VrR;2Nyic^hLQW z@(n{S`Y?K!MBHZp26{|?-g{<`A(*|sh?EuUIQP0B2B3w5;+^pIx*pvhXV>LyOQF{F z_wGkAQ5{Byplr~OFU1c^oiuQoDKBj@BB?KqAmzi2gK=jqFBP4p9w;YWv^mz+e4B#V zZs(OB4g3KYmFY4@8KId$Z9c_2#`FF9T{Y+A!2_CxU#rVxu&F_RD@9l91JSktNE!?)zhOVv|4TCi-F;KQRBo$6*{g}TXNsXMKxq`f2<=wW{VDTXLH4xWf~mxQhgFJBy))Z!;^js_A`UK$0i=g2MsI3ey3YH z4uLD%i-*}7_GTP2&s>UueLN1mNW6sB8o}AkSurpnd&LZYUFVxwY!9+gnl4v7)U3qh zIbu5|w+OdAbxN;0H; zrr0F;V+95#(Vb1AO#Dy#a4OalySr8HfU*9|nG=hD{Mi232TsTqeR=FGaDN4l?VlYc z+~|sNV5YdZ0s}BsfB4w9P7Uj4M=hk20`g%u9Ypl3-yd{TKvbjb;8p*+XB5F8G@+Su3TFBg@wS!T+UdW?YTX0%ueF;(V zvqnvS28?|kLY}$tjv6u>v<-N&UvZV)Aeh%6(9)m7@}>!OF6M)`CdHhLAq`ZhQnBw% zor@JSOuLE{n-T{Xe!w$>d(o>ofO$c*hnCGCE3#n!f^%V_xl72yTQ4h!)QE*-IGAV& z(4;5cD#)*kyt)7++btygr6@vT$8ajA&iL7Xy8zk~8!J4$f>^~W9=OA<{GYP!XSutD zTsz~~H8IBCJXr!00U>6VJHKR9)-#kqIJ-!lZ~(F6CCnV;ABdU!8iq3il}dQC50^rN zDx_?StzT9L0VM=7QlXToGB=3d6wqUS=bl8C!LuX-&lQ-}5-i?u299v%qk~9ZZzD2) z`|vWs$5nKAUmPStxJa3_CwgR@^)T|2nxJXTuqse48GTZVouNx9$UU3LVxGG+bPa%h zm#_KCVuo~mL;6!20;#D2&=fVUeRjTN!BiK6(^~v(P7e%bKHXY6fSM&1-*#Xp)p)K{ zw1TFC<`AxI#n#?NCnt)^$%(oK5&GhPQs>nmbtAR1SK!U8iU^j~mA}XTpxI^AGIpMM z`;4A_=5;3;6cU5;Fj00gr-YiEiM8RE`MSCgFMh1@Oa&d#8Cp4@LVxGURQ7LvF6CRk zs_LXsSC)s*su3{l$j(RvSBR$EocT!=i!Ejg$vReodHypGz3d}9@~Z#08F)p1w|U%U zsn|Q?cu_*K(hl+kH#LGQzCmvmjas(N%ekahQGAfl7T4HSkelXnw2jqy0?@g)ve|2s z=p#neeqeT!WI}Dhk$nqTbegOk3WvuPePd1QpWUc5^Gr%j;{}47YZ+qJr(I^;GQM@> zIGM;*!=R$sPz85rt2_K;8U>DjUAmK)X)Q)h1gsyvy24CQ(l7+BLgX}vOKBI_79n$| z6h$*q1Hucm+nT!d`gq?d)X#M2U(-CA61!~xgeS}JW?y_af zy1=yOwuRJO0dVs!L?gAq{^zX#v{ek1gG`if&V??htpVqoqL@j|eT23FNM}qlC_SOX zRyXP0k0gKs5xO7c(Xot3e)!>bJ&9tJOA-kafm@QDE;~**7>KOBfSBQ{~ zX=KlLf^(U~aFvUp7$VL&cmc}`via&74tgA>>k@MVNel~m6P?djr}^Adn+Hh29Pdcl zlb{j1<@5q9W3!ICAq}o;Yqhj(6^CtlCfv8LYLji!;5npZP{7!KK;JSna2RloEp9nf z8xZ$b8JZn2%`z-@%gM;-CggjB(tt-R4P}lEROfhGp6~MWB9lqoMmoKln+ee=*;HPd z3N23aiwy|@iWG^>%XK6f{{+wac;D%rpGS0~Wg3hI%fwMAn&REm0O!lR|{>xg*F(AQCE19@CyZtV<$QHd# z(ZjS5{2o4G?3=dsXvLV|kPu4

@C+IPl`wM=TUGhIBW72C+Tkva{#>>z-uK^6F-y zL~qF&gi>V#v>aP|{!kr8+MD9Y)3mo96Mp&=c_DV)_MLyz4{Q}e6G z6Xxw=zUE7R6z?$P2i?})8y(*gW-YdWJxqKG?wo83SSp@2UcSuNiRt3N*>&xvdISUg zO+k`b&fp7!h6-Ncw2-2fdo>AJ*QVZzFd8@b#cf114K8WkSoGc(YQ`7$yz08Gol2#t z+Q(eg)k;;nw`W`OZy#|^??Ic@uWIM=<~(RJGOR>@uH%Sp0aFA*rR{xe1`MLioNYmhhLvfgt5MLyexqwVggebFYFC`eApy59 z(7}iJT59rX)37?j(|K4$3$HijwlG63K%`XnruwPMWdk|lbzFn^chr!~o)W&pspY7T zmPdwvt6QS8pZD~FeJcI)yyu1{VZJkuS3fg7hQ3-&n>2Jzfdd9e`mf?3Ge2U! zuY*$$XW;`LAj@eZc)0o{eUi+0i~_zyPB-~4pk3($51id?;ix?WeY@oB??mf)qT1fk9gmBx8%@Jct&p-(?O6%WjX*ry-tRCEd z4?LVHDe{hOjYB9LfEXZOa#ufOio{-nY9mGl{vnq$_~^{CMHtH$LpRle8tFBMO3&7G zZULzr01zO#m8ubxJ=#iNQ=i;|){Uc(#o%YlFm zlfub~zdBA%C`Z_{SDy{HtKp+(ho91a!M)CBXpUZJ8$a#R4@`9D98iK)j?se;JiS_( z73X=jT&)Yw*!Sb5n1yvg#;IAkKv>NWTqG;&D^1ny^8}+I)l^81ESR|(W$zs6Z&urU za|04(S}v~u5#q;w!-;|h|E0^{??q@k6oZw|qgbha&sLkqD>2m|?&S05fC!C$M!11B zhbrjRriVUPf`*u%>p~rq72#M1jMJa%@Rw-6+r6KTZ5CL?HU@#+uh zz5yII*Fe;zB`rwX72T1f-l*b1KrR{3;yBP^24aajNdmR&GX$6&*95e2g^f2fsO=`4 zaJhh8hONMKwv#0)Xm#|A7<@f{WoXXi&^B1vTywMPNRY1RKusFN8E_c3*kG`#X0xJ6 z2D?4h^d{|9Q!8jb#%IpvA2d?2+6q|ID-12o-XF?ca@K=Q6e@ zg}f{&qIq$iPj3jLaVQQ-<3_uRE{ip;bvQZGhQYI9{{MkS%{znNV^s0V0X{-V1*V#n zpi(gt8zrK&;fzE+YSq;w-w7Pp;msmRiQ}agH?bxW}!1ss9?-A)B} z=-_$d7qyNTRmDDZS}L(;QoZa4t!9FTVJ$1Aq=gJDRv3v?^u*|~s@WD0IVq7n8=4mX3=c=&zWamLI+Ln*b|#_ErT zvU-t@>*X3QP4=fu4uBg9-(l!Lc%#ar=3cgO!Uks0!e{X&(q8ws-#~ZBkK4Z{$0dk) z6&o9nJ~g-%n<2%2Feq4e9-4*?VMy@*YPt6?0jFUqFvxwWru9vZ7+90H?#v!1BFC## zq(&Qqi27Ddp|!KvXM63q4ho8EeIUV^!pGKktnFclF{TRc;qrnmm; ziL^%}Wb5)qg69}#eauB_B&b5vA`n_>v75u9%&z?qtwkwL9B?&9fKQ{3jer62qRX1z z4}OOY{uS_le-VT7!Rg_{cXxvKyC44V;U}Mba^T?q9zHz$=y&|z-wgimoxuP0`u+aV zlwvI^eUr9Y(>+jhasun`Vc50YfwbTTk%N0G%$02kCrz{^a6n$<~-|}O zF&0GsRxHPqO`Bp$9*F__OTJm`*55D7E%AQsc!R$#i<^<|c8{c`#ahhbsMeB?Clnq$ z5#LiDR4^6U3|2&j37heP4zF|KO))J&w~(DVmVWba?F2UQ{zQBH^kh8m>!MoCuL_BU zCuqWdP1&@pz}*Ql!)(f#SO9Lee=|% z3&8ka9lgO40LN!7&-AQZ&QJo4`!KRXRWv?-B8NSrIzrDrrC1+{R+8DSRP{1bUUWUf zZ$`Ia>MF~ z3xW-~Q~4PJ63D3jV5uJ~pT>EySfqHjB%!$V9ZR4n+{U0uV8#~!rvu8~)kf-i)l>w0 z>(a%=)Oc|(di&jz??%zl);9j$k#^Kv@B zNu4ibxCr4OSytt=3UmiA-{y$F%ah*NDD7!3uiou1E@ zGQVE9%;hXsf!5(dpo#$&->EnP!=={qqVZ)NF4kNO;HG4WLGNeDhCsuEqA_wCFrFHG z9Cxx{(i;B;XIdWEz|+v#U!1}-&9&wB*NjPVJJiW<#p)DAhm(M7tIZ~yqQ*R0E<%JTJFf2Zxd z@IHU1-)#DjaUR{lm(0`6`&7-qZ~Z0rAn+H1aR|KUGB=MC?>pX6@_rxlPbTk+`dGU4 zm+ZsM`v8hVnV!o#SPdj#c^?vg!p$8J2NB65?HtbU#~0j_-uqg8!r%HsoV4J5$B+`< z|3jw8;cp*9)Oi0-S%Qnd-sX2OY#{Y;lxu0{PQGCXPwdw~Xu6vZq4=e~K_@Bd@)T4e zj-}mlE0be?n8?I8?{sQVojlY_`-+}PW)Oh9rilkh&xrZPFddBf?vWgSkFwmXn2ERL z6tQipi+qg|e7=?0@kp8{eM^uByA9>li`FYqON#UNqHl^~O$P22giLNlrDAghT0PQ) zD5^cZOUpZ4h2y9z`~SD%LWV>D3pgk8xLeeV+;uS z2t|sjBYIGa>VeTC_$gI?0dmCmG$_?JuIB;YjSNg={32ZTIOQy}1I5#eV_+*pQK~cW z<$~!3yoe?4U=^?D`EmqyB%DLQdp)*7I$SY*3Uc^nxVl~zvq%z?(350+%OfBOZ{d5< zYAyY+wdf88i#q%t3a6@3_$nR@dQQwv<>~#$6H2UG=Oem06ms5w<`VKhiGPH|Ql}=0 zJ#ZggBwOT5amW~{b<4_sivS;peWNm*%AD=w0MJ-I_Kg<7R>T90q^rusZobWz#cJoI zvHvi}>%J71JH^MPn7>0VKk-`yc2#aPz8X0r@k*U{#H30IL2BBGN&!NT@NedONLN|= zARi<|Iiru^Jil2=?Y6P(_K2*O(&1;Jo;Zr37+LtXs-#U@_|T~9k1o1p_546_NhxtK zSl#L|hjd0m1MYstr|DtO^&-$!wqW-YOx>*|p9d@Apu3kGIsqwvOh#i>+Guq{je+Ot zibY`sk~Lj-hF|)yx>ONotQ4OV=i~ZD^-+d-cUicW$#joJrzQ|TYpv^^js2(Psf0c<7Qn1G!!L35@vTT}kpa6JK!4HVhff3FT7Xm#rW$nfBa^8geTgj0UsJK@Ir-^0UCKYr-m|9neZ2RT7rN^ko#9ga-Q~7c?nCZ>4tC$`*S+$^q6u=>9EMt1b}?YI423!PJ%muxzh22zk2kNajD zohj^3yw5e1mfy#oJD(Raig*@_FFSI)K^c4uEa!uNdJJFDTs+o3UFysFRR3-XiLT_a zH$-F+!vzd(h~SdF5c8&vemTF9wvJHqPfiRJyVcoQRcs-n0zi9;^Pim5B%7J8#Mi3z z3gf>NS-iPOB`FCeog=t?(qwUIlYTC8*JIMsNhirLtuw^S5 zZ`hjobi2##Mi6{zUMUj+f#fv7LG`rNMBbWzN!Q!ew5b-t?1FweijLh*ME9e^gd%>I zt7Wzi=~`WIKZ3}@(J*TG*udzDwU8vI?%E#-VywR8^kg5(`*%lorS6I3FWDlmF01|e zFbR>#rt96UNs*;jjKI6bb!ClO$2!K+2A3U;RkUwgr4?52{h>VHXVJ44M}i6MmX|eu zql@#aRXGzZ;MMoTFCV}9J_4x+V_*bNgWN_~UQPBu@-gF4>i|$`mf3v*WrZ<0t<_|YUXydZghYdrlDz)(nnEcl z%BP!^I9s6bRX&H45rsQCnI$QSqj`M+8Q4hVFpkek^b#{J={+G1ze=OI(RM6;1Z=~q zz2RyW>r;5tC>SwgCSV5*t zMbeN&ISyxGy#OzNlBc4Y|jL^M%O<8qg1~SA*1U8WMo#IqF?fkWMm_O8l-Gg za~C!pT#3mwn8iT%3pgm&M#>_VfM38SV2zrnYyGb6u3Bd#4AbJBpejZ|1t#GNfNS~m z$7{jM!0bh?88U5|dpCvqWJ6BPSX!}!Q&UMdr{M~Kj5}uTIOsHlueZkrXI^|45qWQ7pbP|s7X%qy?z5`Y>xZd)xLDA(LN4r{P-|oi4Xiw18gTU+YGs_{(RhHpwHcHl-67t zmNbUtX@dCNgq(L*q11za3>x{n>{K6!Obe_%DfSn*G;DOD1W_BuOB|}vhcXkl9>Du1 z2-Sc=Mf3Uw^!RIeE8@~`WuRXWJzXwi9v3=LCkcVj@6lTCLdD~KT{NJvb^FWYJP-9C z^5?;B5#1gWbusfFJ#=2vRkC?b*i%UK_fY0;-IZUW;q4mXFVS3o*yOivZBX;OZAEbuVPLy%cW`osgx0x2LDo6{JtFpM>g9hEu zn-))L{TLgOY;aZa=GD{3*^{S_zkBjDd-?S3(UYUMN7=KZH*a4Zz5U7|YU!)UE%o?| z7F2QP(>U?@p*_TZ!4dYG0JgU~Yeu*4>U{N7F(S+ikjUZ01|H1-I@V?P-zi zsFjQ?u=gflXtZttBlS_fG`9e){gB$d7=^DzP1NKrmI1ql9CU>Lm$ibX;-pR|z~09( za~j;`p63~-kAw?3Y6)M^((zH>*1|B@Qs@yVCJ4KSW|=;JPpQFh6Vw`=%ADnpu=*;wx&`XXAS-tAKK#kg@>dUQA%VqEF*rd{b$bkM0zvWodUZtB+9QG*Aq)TB|V zJUfdQsUvCt$$>``8|l*()yqm*Ezo(x_UAIcyS#Br4+fsvTSpu6Y(-+{TV`QDvq#N8 zkeliBbm8@X+jDIuRFPXT0Gq{Z@gOf%MCg}Q6MTJ}n5Y`gy(*Fa?D4?_aJ$q%g`k4k z+CJu*-P~qX<#?n%oZ%mJ##>cEQz&Lr;~8%@VW`!P{7`S3ltc|j4Gd%VY4 z;PzyIs*JsOx$5;mPrTaF1D5HoQ7ggwO1uIPyHbLGR&?-RiM@OU%4=YJkc8gJ&vPZ@ zU2ynSLtXPSd8`*+Q!z!1;n;pX>vek-PKO&eONPgzdM&~BUs)cIvQ=rbtEbXHRSjx+B`4CM8&@XjuK z%XD%wX)W$6kDYFI>v&JAYQ%&N+E=C9i*kd1Abi`K6t5$VJ5lK#yjnY+!OvbCWskr5 z{@ZV|H(&qfQ~!36y{;VB7o7Ii>UM1ljR3kd7m^ZxXDde+-I3I!sT#q{8bjs^%uHAb zY^>I(KWJUhR|P_N%ndI3MtMh&>#CZUr$n37iyD)GcL}cAc=-dyn}|)Ly$P{Zw3&U87^|cwP2CKMeR9w3V@kAa5@Cx2-$-M- zyQdFW37lu4vhu!!HPD%pE|cTbUZ6?9%~MUd@&BBrAUCW|!PKB_;5?+`_Q;0j9tWB-3AY9Q^S;ET z*=ri@J!h_YI=tfkT!z2ZehM~xT;K0GOuz*pVD^uKzjqqMu5Za98EZbv9q*Z5-4Vb^ zk!L#;hOnw>F81i~Z^dR+%+lx?#!?9yLtGXPCFvhrMT=d3wT-sR9s1RS%&DW?$mKi@SsxS2keT?_$AqzFdX9L_v61`|SS0LY z^>O94I6madZ8t?OIQ0cUu@>*P5DJ=!K~pwS<0ubr{zEDpI$3_1u@G8*y@EGyEAk#AR#9<2G3k?<2NQnZWbxn9Ud@l9NrfKt9 zqaGYLba)bCFu*H<9~ei6gYe0CFM9fJI^PK)26*d#P*gjogvStQp_(A$QxyY6#44ZL z0Ri}KOKT1qT4VA_kXd*tDR|e$2Ll=Orv_nYwW9gH#}bIDmWw5p6$jg9^v!CZ>#)8D zbM(L7{!yn3?JaSZg*mdy3&IkO;B;Q>;Id?X1GwPxT~%H=had!bVH6XDQ;QKB;#IAt zP(^xwvM~nCeCyCD-j(7k|L_0z|Fe#A8Qpgw8VF%tha?Ud>lWgO+>Fn3dvA(E&d~7Bn2Y#P5ffVPGj*HMoiNS;IlY zd-DM1`dDwqvC&IYcn`>|bSld{<*fTXD{u3EzPxPRW;#ws`;6Aw*JuBziViTR-uv&sRBMW)dxW`LC?ddFp?Wg`Q zo+BQCNqDTg3X{@Y?VhB-HuL#)ek1-QI8C?-!Nl_Lc@jP4t1CUy=0qp{oBM8?`W zJ~<{ww%Ll3mE41<%Q?jICu}EOh&x|@nsi{%RGXuoz&q4ZvU}7K#x9r5A^)S%3a^d=h8M}APjUnigd!635n1i}MjxzKQ zJTXn0@2w^$G{EmWK~~sFr%7V8);pW1_Z#(Ao5P)syfU>UO;+Yx-mmeNEzV7UHE3R` zFIx9Prr95z=6$edZjV26dkmV8R_JNC_oA->v~^Lw6GE?6_$3UKX*pOI#at8-1Ew(C zv^Cu!D0TdpY7bFB2745k-RA&sjj}Ug#WC*1({)PUv(A?Vk>H8!fe5eOAV4A`o)1;%s0|Ec)1lkDIhwzj!{{ z{|POB{X|=?(HMj zO{TaHj}Gg z#}J!eGGJ@REfIU}X5?Le&Mj~J)U>J?n68vwFihmVJm3eey!H$nSigC46isu8Mc5V_ zN{0eMj=Uquw~=(r7!s=9Ddc(0H#ZI`xNH&9KY1x&79S9Q#Hu_mwnOycj--^7+K#^f zN6^A+B{Bwn)50cujb!+g6T zUOWKQ5m1c9*qqJLD>^hv#%S3iek?vR@iu{PQEi$Fy7c4nI0Q4gGXX$ZSXB7 z1D(9tVs>@?-Vs22&}KyOf0Nm{jwhkw8h*K*7i`V{`Y#!qu2?%Nl$ zZl_*+`tdK+3x8R>$Gk`iznLDn3MJ=xe%v}i%0)dqRvqqz8d^m|at-+syN?&I z+Ui)}<4ZHyKDwDHe=HHY2`0Jgo*oAo1>bV3bL0JbZBFu-n{l=llR9A>oS{ZbxHyG& zOG z>swhHelmXjoG=+0esUlE-0!i&@QQ-EA|aX$75cOCy#&!me`DyRELu)k8MN@(P`aQ% zp}Mxb$s1xpL)0kWG1>L$S;SEH2|^A*CYP*&_2_`%4w*27H31mzZ0 zqkFW3yLUlm^4zr=fQ%XnSd+XjR8hc#u`3xq$SPo#2qOxI&7dQ896udCEVEN(XtB%#|a}E z0#}iYFN?7X$vhK&&mdF#1i^30f2uBygTv3%*>9|QKb>GJy#_QkyVu}`!eQHrLD+Wy z-mCM0j(2ZWs7oE4;u<^U-ZAXoGx#P^(Rbo(>xPFDe=lAcPqI$YEls^P&OFZvFnD+8 z`8w)7%Jz!mX+ealPCSi|6y9f3vHl3*mW3D?S2)PP4hO%UDA63p=?WZsixMi>Hi$7Z zXhfiEz7HK4O>5_??=J_mdHr>0vd?wv71ImrmV4X_cNG?bm+m9SGJV)SwC}~>G!=%) z1smCsf6+B~=d}$&6T|D`uwe7MU?)uo5EC1L6zzSW^{nvJHk3X8Q5bg5ytQZn;PZoG z+X4BkbZ=dN`S{20jm_sr5_?aQus6cE0R_hLQy2%ZU>bZ4#+{rMSpSMK`;@Qf4?yru z7gR%zF?8BRLC3?+(1$^G-<_$Bo6n7Q#KgAne}W)FV!$1|84=Nuom&A9Ic=jA=8%)7 zc!!H|?RsHfyD2w}aqW=dKxT2y8Dsu&0B(`P4>7L`gMf>^xpVA4;*tkQR2UoFM&k(i zsnmi%<{A!&H_X zf8^?VJO(E3Qn4qR`mc#v?#Q;ZB4+4{wZ4f}ZwFkVv{mA41bOv@HgDbIsb+^^a_J&S3t-xXUbhh_eTWQM7f6@S?j!q4mwzMoPoO-va7N~m5)*9WSYvWL_ z+PYMut^3XL&`$Q^N1e3baLYPv1NZW@6XKXY6%&331<=O4(czM}1i?#c>MbB9`+Ac3 zL);zg21d*VF1DuJcPP69fR+xe=!&e>9+f${%^)-fTTXgyj_P`<*#hQ7^Gw_Gf3|I3 z5tSO14S_HftwzpPtC;~xI?e}#y4;}|P* z{KmW=*hcQ@2N9xj7mg9P^@wP(qjk~!L#l0ulk>L73vK=c`*Ewi2_-BEYXXxlGq|s# zM3?Df%s1}U8LKKTyo;}W?|t9@!uO8u*l&MD{EOc5{(WxlZ*{l4_vNge#|5BW430g` z*J9mysX_P3&C+xDcE4W-J3hR<5PU*VxLD2T`xS;F)Sx!24Z+dqCPlJIb(q0pLec?L zTLia}K_P;0wOd0zA{qK%(;>KA0^(2Emw{3NBY(eta`fC9)~N4Lc?l^6D)EZE5+Zc< zihruR`SSo8=~nY53@cWmW>a^57Ts_uN$1MbVguNpa#&OYtiih5mZWTa4lmL*WrQAd zNil>t;tc5_0Gf;U@u0^h{H5ee2gz;n{S12l@;~f9K$CcIb@zPXEU9Ekru{rF@2SN!Mv`|;typFUF0==tHthlij3OLXvS48RUnrTF~6{LlRF zUi9YGlm8gL5Q4|DDu!Pp^!Zs?Y(~-18Zkc}rUyM)>RCB29t*K^Q=VUJ-xlw-aKg!- zhz+zUPa#EPr9j=TUc7h-q4Iw%rrRFEv42fh^Lc?Wja8bTPE~%qm*gsxL~o>2L#5$x zK>+zeCz;fbsKoYWjgam1@@OfJylGx-dp-FC*-)=nXYV7h6jk8|@#rx=Ng}u;rreSl z6weHxm0nviN}c*ow?sYYUX?zc0XUsUoHwB{we1sY<>Per3@35}y?B|CLG z@_e(}t*MpyY?g@us4nu2Sej}{ZGTsp9+h(_ETvSL-KfxS>G7!$xt&KUwILxQW;GTA z4nVK>XT3FI(fYp?OX+ZgkC-X_dU>`Y8FW!Dv*~#zg`yEtRvgCu!}RFIUylCQ8fj33z# zHN9hVTJr-MSI%?{I3Y}|Mt_)O;HVZlPHgbhk->a%3c>03xB{RxV{^;o#1@rNpr}}6 zKmClwiIYN%2WTwfJc!}%V{tr85{9W#A8?vz21Kof)4=RSVtk9pfMP|@mieMcq;KzP zwwo7#&trTA!QCs<$FRgP6Gb+$QB5cS20tM{t*rh;pE56c&|auXVt+BA*a^)Bz)ZjS zL8e$+NpMmkDp9(qbm{>KmP^BkCuD)kavf3AKsn9Dp}boDaT_fQv6b`};^N$1+&&ffJRV<){JZ%*AH4go&wTCvhsztx*&_ICckXq~uKE9X=zF#!$iW zSMw_|EkwsWal}yzysCB_5d2WqpiLbohU^06Z~|S4(b4YWO;Nq~D8PU4W`{2!F?UHHGBZzB~n!k4x(E%3O$P zy|byHM8{*BL~wcXn@2M^qK4|+lT4I^ta2l_ix9Ky@718)n?e#&Z5B&BpRbnZDDW9O zPA^&6O9F^V_tFq*@wnSjAr`W;R#Jyz&#>LS{{+BG`vX}yOWa3z+KPcFmb*o<5&W<2 zR;SjEn12Hb&OPQcsZpY5#elfWJiLAFk01i1U!7FRiYyti@nT?DC2u8#-iaidn7uH4 z8}~drnFb|M5P;Sp!P$ywsuuZL%J9Ta=>PFd2F7Da^#K36HOGr~buH`HWDy%ya)Pde zP43`}IIwI@-slgCGM|jiF*0RAMzZWN>&;PcjDL0pR=O=vh9zQ#d=fIGFrqS39d?EP z)NYV91a5B0HvkX}C}xP~l`k*RB!5xlvnu-$Sgl<8cpXJj4P;sV_aPKn=3zM^Fc^l0aM{-aLRUC62juzA>SqT|h6mM*um)yMa_K9ox#9Klehe(B*|t z;eYsGU5fK$9L46^NXc(yUSa)skVNeC8IABjhk+_EwsL7~Hg8(>t7B1qVp?}_d+}&| zW=oUE?kL+%?`-IR9B6E11wbmA*ag?v?vr^GC_<)6VKZAgZ&bNsXL#_O6XUAK6q`43 z&e4aXXkAV(#VV9JO9(9+ZwJLncSQyarXLFKrxG9 z6JhKewTanprz0PYqA8^QF8s}aJ84#kvu^=PL~V>x_N1P>P%>Y|ER z$HJ)~iG)NPxtNb1`I(DAN|{1QM& z!C54-7+{nyRx&|w>l4J(UgEwT#q-7;k$G?ID zRm`8qtVR|HYW7Ucx8)hu@T&a+6>FCkYtaRGY7*#@E#)*PRdKIc#A(xh1AoIVlln)f ztg}X_u}ZSca1OGp*M7y*EfCpOu)R3L z{+qux@#+o#Eogu`E3=Zw@R~zqH+lT(drBn$GQ|1z0!>M$MO3`YA$brX;7bcH7+-7q zN>zX#1ln0KRkO;DjqZLz2!Ff2D8zoA4g`&Omf^V&+Rco21zZin?lwLu(t+k2*VOP#zB(|>!J_p4~E>THD$ z8=8Gdz({D)a+*80!_XxAR=A>UCbl0%(8`$fY_Yih9kgr#v(hGe*HUI==jL29>SMvZ zK9iPW*azLryZ(i-9s%e2v3^b;nMeii$u6G{YbGSQ23b#D2*KrXz8zxflR>lw5cKO+ z%^UQU?^<37-jm+5vVT{Y+A^1hZ?v9G^9?4b2QpwVJqHbvwD9PDq%RI<#XHGI&|lNI zR5X*nJT$d9bEe3}NB+1tvmb*sw63^};yxJb#X`x>pqqUmd|MB)rH~I6{ecB}A%hSG zzGseBtAz;c?4rcmm5JNG0e3^JL}YS4H-WUO?DU3~1&xaJ&VK>xYT2&tp0c>Y$fvHq zJPK8+4YH0A-?U#gnE7FIn4fg5M=*JRE7!5rEVq!Uv-_4@yKEXX?D9Q#FZf-AGv)rm zR<`EU@}0khaHHJJN0AO(fJfxUY&>*RmYuJ69HbkQ*^Da&PT=}fGrpq0aYA+Ed!vSd zHz8^%k5Bt#N`G2cc;`bYl;5k#9wm>n=^91^uq)Y4a&^|L3KRWyC|Xu3^w0y_KaZ@k->uw=+1Qr_9S5i3EDNRAX688eDbSF5=whiLjz9LF@6 z^3DHtd#ao=u;EYy_E#NuwyCaj8mM<2<{&667&+f`E`PI;9aS$57Jth5n?k1Cn0d&2 zp-U>lQO|{*^h%F|ugq`uJa}MNxU2a#Zlqbpm8OMdU7o?gI&n$Bhkf>s=GEpSfKEbr zy=88ChttXdb!t!2-4c#vWd{J(Ut%&mojfsoP8i$>L4Io9bX-nYTuh}~+buOqb3syG zno)GJ(SLm3QJ4fZ=C;u_e5h;3?xqp&PTZ?U6t!MhUux-k$u?TtEO;EGm zxH?@p)}0g#!Jv@vFtEI>ENCrf_fpXIFYmF6&3}+~6`ee|*Dh{OH|313g;5b%l#r+C z&0mflVB}xzpUS6Zq&+VQhSE~jr(N*1y=$m&62)-v*)sKD5Tt=x2~~w{AD2nwql+^B z7q)@JAJ|MNE}J~N6p9S;-f$M9KUJQ4N#C!AT-EcE9=3*hJ84Xjd=<-y2%P zw|}Jl%lAlX8frgC$Sy`+)t3ckDxMZI%>9WzR$w<2%0RxGZ<`jHB^?F=Z>{M{*Q<4S zaQhZLq4!$9_vmflYzu@2g&X8XfHT@k(zP<;pLb51Zb5qU*q&TbSI!!TUW7>*`q0?; zJ6u6Slk>pj&4~+YrLeh0TB{pdzrvOLEPt6}lkJB`(l`#rma^Z%OUEYf4Hx$1@3H?V zTwSYfWBpgSu(f^0%$98!b61fYdr!iZU2SOWJ_uL+@WWOfl5{;l1gGV-KbAnN_|XYz zp$jMK$jEqU!$msA0H%pt(GzOuxSJBzkhQA zqrJC;KWbo+jUu@L%jxBex9y|6YQ-SnTVJxxqSmMQ>=tMV6$8KCftS-lO&>8nd7mYi zVL4N#P;}i@Ca^OKX|7B=29{!@o+sLnSb{{BU6;$*>Kbj4VZ%{_K~3KcKc@Can9%s$ ziztnpQtNzaU)JtzOw?u`=oU@UyF~tByXVRI zvqE!3BaF|H+qJnC>$wG5#!RDvLllLYH1Om9>ar=0bGg+nuyG$h4A(qY<$tVL^}Am5 zVi)a(q_9?{V36Y0p9;3VI4x!sb&9*-1%sbGukJ;5yg5FcM8jwUxd84*hlA)#cew`S zaHiSZP%E$2jTx$IPW`c*OPFD|a?gU=GHdnz<@f*oDe+%S$b<)!rh`l;w}pdg!GHPq zPY0hk_%DAt_~>{1mw!t9mw#Izx@asHalEW=qFnILwGHoL{z#WgEV48}Ulg%oc~y0Q zg7J!^Agz@1+yr>hIALJ>F=~i0*sdggk}A6e!yZ0wL17VR-HfDBRh3J%^@ya!R$91s zA6n!P-0k75DOMNY+bag;%A)!P9*eQN>PC3F5{GF8LTkF(ZNbk((tr83xPDD(6P|b& zEMZ6)`^uNP=p)`(6Y_Yyd%oU1LZ9g%bWDwdrK&O4_rcl-$&(R4oUHyeJbDCA!f-W< zRAMGjZ&lT&^bkAR4eLf)7!`ph^pmc4p~SC(nB&nC5;E98t4i<5*YxDgo0s8c(ILdZ z@94RHf{sYfUw!{rI)C37uYoMEsb{<8w0d*{7gF>Z@5_+13CII6G@I45sH)7!QI(zT zD)7!i2QLdHK$ap1PXuK>gAav3iL74O{v8&EkA&^PE_{W@uQ}Rra2})#+TdHz3xM{u z#^*Eu=QY;Vpg-NN1?{3AzB7s27esT;>h39)B7a=|6Lw=+Jsr%^x97 z*{ypZeNc@%q~4yEBj2LZd@lI5jfP{%FWh=OI6m}A2-{lC-Ot`^TE+Uz;uaIkC5;OK zkF$y0(7@H4Tb>Zn{niAwONE*@Ckyw4WLyezYBSB*ee;C1EGIKwHF3M%Ia{v0ZG( zV9t-(L4Uy(?$!|dmFl)zLHXFA#Qu6BKQOR*nL5ZNA#dbf8_YjkooNJC(bS3o$%mVs zI;$Vzxs0iTU_u`~R4p7!Kv?*j%Wg;6^=3;xp{Z*yuYX&V&CAOoZt8&Ni2=fU`h!p? zGQoVZH?N*P&YnDd{N0nM*~_PIkDeU8J<6UPy?OiU==y zkI+-{2!92ONQf>5Eg3)z!2c@JWWB1kQZ69`poRFn*ixIDM52>mIDNG()={;Y_RJ7@ zO6dj0CIuyj_C^97pS?p<_)sWZzY2ElSx(C(Jw5J^GwhzgC>nA(I4 z5zt*9jw~1tG&8h51=$cTXL)8&YMNw4!{`B%GJnAm$;3#=?_L?Ux1OPmepai>hrlXgHxw!kVN_%{&`k?y4#yn<@2?k*UTu|VaNG2_n8_;wF8KoJItvtINJ&;A2J1G*s@_*Pf zBe}lrEJAJv(#9}}{j%E>a?y!tZ#9-Qgu{^6SLgstIKOx>4H2G}6=IgA>BP{Af_y$W z`mlgo2cO*{&9=9YJVL6NGe`|0$Gzy$YI_mAd4jU@lQ(e3aV0S_s~Yhk`tra6o~ka& zv+W32twW7O{k2j#q?8|@oKV}%&VS(OpPUS$e7&BR6cz4JJ2|1`Jf~%j8I+A91(@r} zNhEKau{{VuWVGKQ7QcBiqATU|-D+3C4fM45T=aQ_YQ?Nv3^zb(E7W^IA72X*69b!F z|IvGR%KUIBT8N+va9w~Mgh3L`s)|J}J~XY;7S?I& ztw4N)EZxXnFw5Ue84N&=clHO4o*p{Ff^<3^w=Hmg1-MbmJnQVZ9^k)YD}?I2oDqK7 zIr#`VC(k6bp5##pi>N*Z6}*`eRjjmWp);{5J~d!|-#a3ApR;%GbAQ!79zC%k)BBDU zrh>wVEy^;Iq7>^`6py{dMhp6b9LF>Oo8>y@^pA0=vJgbfkU5RG)(D)n7m`pTYL{_6 z=q|<(cvpvUQ|?-^F;skbT8GAQX4$5;DM!?B?M5%?cAIcV-{dwO#+de7YD)4*YBPW^ z_fI|c(DOue4{*`jxql`n=&~=RxGYanHa|R^5$u8tdT>*FuFB#X#x+>qiLec9X|=`5 z<$eR}FFn2!W3#z>zrUMwUjwF80-aTU$o!F_V@C8TVJNSLqW~Eo0tb-c2{JN#KyG&; z>~qpERu}FNslj*K0B( z2ME%i6L|kwUTufPyK<`$jI5hGf?SiAF3}g!K?IOA@~0TUSp~Z699P0}MVD|Fcv!>w zH0Yu1ZYIW|eUjTdH7VYR&x>nsN} z_u-R=Un#|fqW`RIFB)4er)KgVhACvrHLPDqHnQH93nAu~(`Zp{HlqKcsgM=`J{6(M z-VV0g=RI53mrb9pDugutuY>fXKR~PAzWVO1-Z^4>8&gmT-4+j)fz1cSntAxirv|cn z30M~~xqnZl(^+{H*LO--W02G&Iyi3~1IdXk!K ziaRu@<6PX`xU{YiM=($)%dLe1Bl|8K2)4iR?SEy5Rz#&=RQJUI&oE3-o7ez(^U1z$S~fO!Z#=Hmv2 zBUt~S$Hsy)Uzhw}^C7UU#so?j54s>0nFK?{dTxX<7gu~QdMp!HV~&XH)kYjEQ@G+Y zqJQeuxkLEOAfSoE0DRK`lT{r0>v?`-TiDFXDZ$~n#dC+2T5AC*O z`Nx!KT)_vL&NbW7WA&UMZi zCye!vqvKh_aaY!E3F^aNy23u+x+eJ|h5LK-9jrCq7K#n)tI_Ad4`AfaTC1FwROb1U zZz02_=)6#MKnEj$W$R*Yv%0RN_`WosIi!}#i(+^vs6dn@_~M~{2vR-dS28+-wV3dmJs-pyfYXEn9Mgve6!?s3 zQ`BMPW9XEX9QmNFvh8F8(IPlR_kSfEY;&AH8b9R%SmYJxHF1!n*sz1>-|B|5*N*&Z zYS?vRC0)by5gUcP>nk)?6*Rt>WsjoF^$@na3`E8b1ja;3)9jV$2Rn%tA894FZmw!E{;dwi__@3IPdC^q52@Wv>Y8 zVN0uvd|ezLOt>q85trw}h%q!8kAN@(PCPU|CV0>dI+W$*(iEhtD1R0MwZ+7_AbSs_ ziTTy5&pjVED;NeYcPi4+Oj*2|LPQzrY-kI}qYJU3hM10*?sR+sF_V=tYmlXRxE9I> zlYsOXvJf-tkT1~Pvg?alJapN;=ou_O90G_L5=+ln6T0*0vHc76e7}y#qsm@ehXZZ| z?t~*k)N#2DaKy$*Yk%yxC2@}9a5xbo(bkbm16~Ssc|i9Owg(MHh~Mk3SA)>8m3p2DtBWPk?3X%Zg|06TAiNNob=p+lTfZ)~0wld677C^>`2e*_}OGLY4L_w|@;G&kJqQvAvB9ajFaQ2pZV@}KiG5R-4RYX>nZR@(*Qzm! zUKsf(PIL%TsY4-Saa4ivNE|kUp2HRxQa+Y~E@L_#PkM3^!B8Uf;W@~CkZ4AHQ@PaG z?Z86wRX&TIzJEmbfyh8*6R>N2VH*C78-7#rs3|*LGbxKAm5_(%*f;wIh0ISTP*ila zT9XkUBce?5G7@bSSz2mkA% zj~{;WJO0-{8UB|G_f=HtQ7M@DcD0&IaIWcUJ|`s2Du2ySry8^=pU)|+fJ_(@zza(v zNZdGk3wPl?8z?LY{$&P~)n(?Z^KZcGqd0ouB zY73!`*?->vcb-ug<v*H`3U@`3@gu z4u3f+g0CS->$Z5eX6Z{(2c1hEb}splD`}1=J$Zce_^YRe#}`-Y+wWgy&tHG{{i`?J zTdEs=54GRCJ$m~zd-LMEw+$pZ6?*;MU%a=qs8=zvUi6_+Rfu2WeNG_12Z`~91Uz&s z$w+8KdS6&$)^qcZDVfptnbDGCB^ux?41Y6xx4InbkJisuVD1_pS~*97*~=9VwtM=z z64y=J7_RSYYKa0M_RE`O%Y z{n0}itWyttU@(jYY>tREWhRj?+`ZxN=9bZ^jzJ+je2hSE%vu-@nycG1LSqrWd~3S-6R3NqU`A?d~z*wX~9k z)=t#QU5~XJ!fG|fNrtr9eFdJto__?bGsqtco#rJ}`(eU&&&#b6txirf<&%@>@!KPnYv5U4ZKJgizkdO2K=e$! z1?%IR2f!*owvqxlFT@-M`gFS;di0fr-JUzYHLez3TqX>t2c_x^WpaVJ| zRF^1Y2g5&6QE30?)L0-_MOT>jUZK0>6?R5v#!|Ll02@G#VLrdkZz@dnMF>DJ%yQV@2t7y+yGjBN*a`kP!$1wM;mQhEATF>p zdb5)4`SypepL~5JC~dYXSvQK*10hI*YApokN{t%~CUE`=7mMr;+<#U-1ylMGijk$0Vy@rVTE%h7r)RRr6c7uhwzra0#4Fgq9mQ^r3NuL@=DHVpICi% z8Va@1vJwtD`9mJ8C?yZ99f-x$54)H$wh6JOruqLEHNnT~PXi|y3YKyP6CjVxh0t$M zHj0OCP3v<>%?7OddVj?Yn3ya4)avZa6$CyT_{F5pn6C1L(NMp%=&(O}u2`}YKR+8V z--6xi4QiL#{+Z^{y|26ATm6-gqPt)erqe?Y#ef6QFbf3lz5v8!b~D0_fx9b>UV}pk zta9)!K^`ZBPr*za)*W3I5XgmbWIYoGLa55a`D(RB!xiB{f`2|n9nHGi9ZPEL^& zAtlBFZn%oYdTUa{&r5Fr=m)g#1zxiuvpf^5D}wo_gf27WIrP~AJy0!Di^7OA1eU88 zwqC13(q|SDmW2@-o7IFq@L3&NCC3U6O7MThMT`Mis&-PJYy-BQQO6~hxT6m$nDa=} z#ER%s1r+6mvwwD{sBC`0u30Q+_FgyW-M8u`dSr|h1>4LRFwJe$^2Na$G;AY5tsWoP zb|y8o4R1$HES0M8RSkD+t;{<{s^oeg55HvneYjrhr=jL^Q)T^Iw5oc&Ogm| zrh$RpeYXEW3S)}(_B=v0r@?V^m};$LG;&o#am(SJe=5{ap(!ICZvH)f#r0zv_)qxA>i1`vi9 zU7g&otCwI_6KRBeDd|n?Ah0^-@N6@*%v%va|m3AVn|So7p#dyfB?oi zuzxT~B*a3oy)KHS{3QXk`csAZbHaBD#%s6Pv>_1g)pq4joANNbQbYDKeRT2`=M=Bx8^8pR5}85d0g4vpk( zOpqUg&(X}&t6?4^0K}uD0HE!OBMxqQi8flmaPHxl!f0gCr)7-}2u4e}FvY;gZGVPF z8S(pAoO|{T8HiV42BPfLNYEbNKT^v;QdJkLqTweeHW{>{DXdTLbX5MADsw6rb)ntp z)eXo#U|EFmK#zB5!<&~C#8p=d&{pXxI>w8l;mh(7aS+u?F&KamgckDavXZW#8XyFn z!G{D(JxHU+dZ{C7Bee*=B30{p%zsdt$9}CPT67 z+-Y%y z+`7(DXV;ZS+ly5`lacy_>NhLi$^8O>o=`r4_e=Q-mo${0t~QWzSCYZne}AmZi^(*4 z3b&GUKoIqdyh1X7u~#f1(o|x7%k_RQT3_5$N4#aUuzx7IVn$%kgN&zeb$SGo(o0d9?Bg>|ks`8t*7L#6Oy%UNLF za4hdDxpojXeCumq^H#$C0yj84-wO;46{gsrE$5urMixi2z7_vuGk<7{Im_E442h!c z3bz_xmFIYzf_c**gn6xp3L$+(v#)&$Av$0SK+|H9Z&qLiB^a$L5;HcvOj-NRmo1oQ z_Z&!XwRGkbVU7az9F|Z1uB<+n;r)icl3p33))g*pb9?S5AcAp1J zUzX50;!E~CiMlJnHGgOImR)lDs`AE+ydj4Y(bVB~HJs;w40tV$A0gJR5i+|H0)r56 zQOvF2vTw1vDuz{lRzTj3xf$Rgl!%qipCkrA2)!Vs;RuTzE95AU1m|gVB&3%5K2WTZ zytMJ$>;qQc+>NBhEN=v5iX}uSP+iYAW{Va@4nTMac}oZhKz~(G(x8d3907k?v{Udy zWK(unAFu7VzH`q20KMf!hqrD4BS%YvZQh@bCv@9Fk;io~f z8*hGt3W```)nd*M+s6&VCPabm!=_~{cG!?q*I^5j!k!{6YuF$S*IvVhdAlUBmSN-K zR26%{&bUm;8RRAmZDpyCZLV-%6UHO95vV$7pznxL?SIYGXX<|CH_#rOq0c-+b?am^ zW@6GSc7KylW^3@?LLI>h_BMy;U&oHdqvab)g5zE2>Z{YSJ00eE?%r=1gWdD8rAQNr zJ+56T1e^`ICMUg+Xb?!_VuP$&mvY*4Z~o_qaAW7UU%_z0OB%8!`SIjvy_X9Rz3sp+ zPKdV`LKT!I8hksfXgREQn<*jQ!EBdHbg{7jn}6|sL+EI;vij=^Qv!+w2L5{#L2l^? zpq83?f(fi#ZZCx52ij!5*_0+$#4SZczpjUg2nYQ$*Hon)XdK?q8GLFNooOYVMhbD5FZPB)r z(|?j9yvc0uv&UXY{y}6iK=1hBJ{mlQ?nU2=4=*vmOWH>1G8`FVY#m0m2$w|Pe)qO# z>d}^Aj_z8dv1Of~7gCkR&2L!k>T#67ocLu#8{FzV%~Bah6v$lpsn3Y41nEtCE+6lB zls0T8pJDrqnqkoXM(Mmk>WyKqJ=V#+DSt#xhsQNvf5z$#THTak%wZLQv&m3KIWa-^ z3Pn)eBD-)A`UMBUXuow4>#dhIW?_A0{z|ltjQ(b<67C;u?HT?5?Op3`8%LJj-+YRW zghANrCUwbRFrY(Y?bwqEFo_L2nJgd(6eYH#-cTfmO-g&A90^#>s+ep)PJqJ zb#XGYYhrg|k=508Id$q>ze6@>#2BEQi(|f#$_3_});$Y6qN{un;J!M>+T~zNh7rO5 zVS2*9NP;>h^RTv43kSu$;ub^(A)PBcGTP~_)6U>b8v9b-lNeKUUJU$Kc0R|vTQW?c z6`U4s554Vs)EsN2L2xQ$ICFOZ@_&S8HZZV3nkp;-HAh>ehFi_RPosF~c0L7@yY19r zFn|+UV{O`DIx+)SQ0$>D#qKbH#_$d70VQd$=5lddoxz3=r2{@hBLN;>ekLsN{e6;K_#Vs~150TCR)4i91h1|w z>4U2ll2$)?APNOnt8+p%C;5sYV>_2whS~7|n!{GOuRr{L#9M7n2N>1-Zgw-u1s_F` zlDDfF##)?HJ&^Gjq80&dJk9fzP$@al>lKI$)A=YwFoAz+$h7`n4+)wuqkQaaFMIPQ zA41fE_Dx!bYvUG_=n5$)ETCiy9C=77g4Wp%KgqtlwYT4z1{*{oiT;I@n zfdqrc4!7F%_jW`w)Q{_JS!8YkG=e-ctN?NYuDLtIZR(lQIRj?oKy2@;sZS3;)QW_E zhSyN3hpgIYi4PH{p?`BCrC@M!25zYcb-TZP_08+=-~9CBtCy1>Uj01Le>hW0$PO+p zn7H9BCK%wK%cnWa5soMn{?#>kJ?yZYCXyao(~SIJIGo3`?4WStd~D=eR3C|UMq0^q zzGQD0=rnZgpk^~fa1}_AyTV*9D(9m&`Ky;yhDX4aZmOV*n*4)w_E_8Am2!^yi(+J zL$vIDo51t(u)Lg+II*w6?}sLU>4gYI(1vjP;%DOs@PDFbHupK7X!o~PpfU3*Cl66S z`E`T2-SyGfcbOi0;H!g{S$Y22WolA+{REq(3j|^AN0MqCj`9Q#I&JZ zh$y-&XElatLs{Bbx6jh!Go*>=tfr??!oI0bA?s5tIo51IggH*6HzYosL3%Vn%-V-k zC(g3iW!=qwB3((Z_<(HJq`Yy63r2c@5a3ymP=EI@rFnKGXNHn4*Xs%sA+zUm)$U>o zg?J!JMh5U=;27r3LR1@d(~n(Q&j9Xbw#0RxZIv{4Hl+eU8G1g4y@>Y-U*RneVz|2F z#2BD>ieVhz(N!7k=_$;ux|>0vNHy5Oa%Eg_H+LFjggnL(6iP1vM_vVyYl`xJc(`OH zZGX!1iRAjk@_hnMFAS^B#6-b5e|AM9Dp34aKQE{`H>fo~s5wWdB~Pd&SEw~#s5RfG zJ?E#BjavQ?s-X7QN3&D6*Z1tFtr7{-bJDo%*Y0qhN}9QKp-JZpC@SeZZfzOOs=Z%a zGo2$vY7ir1n{^sxy%YIHv?cMw@4y z5%gmmLeBco8*S~O-u8#4MA6_NFW9Jn6`aezMU4oWM;$;Q{uC=Mn99F)n11m%NEE9# zSc><{nsoO%*WTU3VfJprA?C4FAp>ETOfMl>4{WVqxqSSJ_tkP+8z6O=mzMioP=9i` z#>JPi3xKgvkKTTu^N;)aO5Kpb@^He0y2ITuP;G47PIvj#1ES^=UZH;dVlY3T#Q|2E zYb{<;)$p0l+E^^F!QJT;cA#8fu2~Et;*nGkdaE|rfE?*67v$P>)H)}!I1b#02p&=3 zKAS(xk*B&h1KU>{sOJz-E!4llw13}1W6ALa9s`Vysw)hk74|eZc!&a(>{d${mhL<_ zZ=d{JC^Nz8{8tIAYAfD2myl31A%DIDL7Zlc zP`xonfIjop7h|`#CWQL**2@)E{mT*DQxIvq8v4Qq&H4@bW3=m>%wy*z@ z54niYO4v`;6?-1N>!tk@6K=L^cF2~iQPKr(YGLSG72I3$h6gJRWAQ=BR zJ6EfFnR}V~rh~{8#>6doj(^e+*WgFQZDL(WslCEuO};M5;1TA(=x9*n18SwP!;wT37p?K`=KYiG=N;NHxoHl?GbCvseey`AtHNuJJ0k^jv5e`Sx#sS~aMmRk(ZXzD2JLW6r7-cY0o8VG7j3MU zByJIPtm+1n%#msQ)w-7@RBx$k2sL;|0RU1A1nINU^PkA*iMj-BI|mrE`T+Y5J6GU_ z)4y=Jiln`eWq*g2jvRMcreP95t1gp(-T|Afnpy7Pw$fvet$e5u7@YjQssSw~(%C1F zg)!k2AkJTae!n*3z!Z<#WVAz%&-9PIUPB7GZ^-dL;Pz>;oOYvL3*4La>EXStbf<}3 zFTGGMW?NU>MC{BiYv0cK4xRN95Oi{+(G)#O%afboUN%Zg)p&n#C>%o8J%n9Q8V-By zrjWFw#4H_ki>di^`Fouqly|im64=sGoze|w>ujkA4qEs)6O4&;=;P#JWuXaY<@EE= zg(jCY37scTR{BUWX;7riSkqtamdgdv2&z}{qvl7rJFJ|;t+3lj6BE?}M1Wrvx#t4_ zJz)Rv_d*787{h-NJbm1aJ&YZUh2Fo3%3DM8@4k}0|6VS}peN-${EC9aG0q!{8d;qL zaF5|@9YzjfEuzQ6T?_DRE_M#bxPwUNVB%oVJ${3|p{1oBWs03|j%wIWQ#|QdV;}-3 z8eOL1k-qJ96e+`mu!ylvX+l^ynW1oO-6#0gCbOFI$5?+*%XX3bazgfz#s%e&M$|Q- z%jhqGInk@^clB*AUTm4(mZ%vOdG~Fv+iI%me!Mjdvt@AhHMUOP9st6=}p;OCKB?F^68ys9@x9)CDC z4B}-~W3lB-4$uTo&eX5dH3F~en_y))3ZN!I8jXKJA2kJpj?ApQ_x3eEDdZGgWLVoo zCpKiIQZf)f-Jh?eYI?Ur$C9$L(KyAC@JXhOV|lu%%TbzS)W)){hnJ0SNDN2c(E=R4 zUag7YFlJi3UYMi+KYB-+2$vk8{ruPqU57qD;>i|aJ+do&+^{P+RpMbhX$Pau$3ZEL z4uOAQ(DAk|tqLj38g!G8ICr5I(mxL$`QXa&1SdfNN^(k;nJ@;Ng+Dh>K78O&)?We@yk(^jKT*;x_s?XvXE)P1QRI}n+=CUPEY^OmR2kPCu_<|k09upc>1lK z1nA2C=&qZbcE=kBJHCxb)^=q~VP7Gi;SZD38ybWf;4jre=?^&bKK_{F0O0F_T4N5Y z>2#p1>F}B5wTnuWs-5483w!T z1U;VbfHp+R-eI$r_b^E-m&)KJSrbtzFGu|Cvk$mNryJ#s`zs}B?n^v>kwk`P>{M4Nx4kvMC0 zMa45AD{; z$*$4;_0de`HgmUhCjnrmm0UXc?$W_`$Ej0r%eJGoQ*rLk-+X^pWYgJ{YOXO4;oNCT)q-JJoyg(4ce_e|13nMKvz$i# zHQj6!n}I16*^V8&zau+Tz|n9I*$?@4@b{66uA$GDXzj4HPsslj>C@|AYWYwX#lFSU zVhZL;=mjlG#W;We4s1ThGlh2x!7#+SOfPjme+t#U8PDJ@Pu9+=rEY&4XJc@6h7Dtp zr@dYNlC)WfH>FLT(z*!;mI|vgm)926ky~|L_(Nhpp}HCQUML7k=+vSi$o?`6OC6k% zRRy{Vl%71PiG7r2N5YeWrdPEYjQRX;xFlEQ=5jgX^q=$WY+ZtN#O~tUDeC%RVm#&0 znf8tsQ^TFG|P7#UGJoF)uwPZhMz#rz2g(Xj&JR}7n6-iSP!9O zJ41(@zyQ>w*2oTzy-osUN;Ym$sgQ#LA~$$lRU-)5A7}OU%4Lanm2i?{4_bA+eq1KB zplEtSwSZ6MPNL@<)D^||16Vg;0b@aH{Y8<@U^-)}p~SWLiU5DV?OqQeXK8wh&|V9# zkd;FxWAE=+V_3ei{bv!e#?xku9Wo7Z5uF_$JAV~oSGq7~;D#?t$npO6%}5(R`%Va(X~Zr=V)wp?*}&N`cIbu}Im`7wT0#DmGQy|3Xq#ReF5A}wz-7!`D+ ziTghFzg}Hh9A1$`I}e;uA0wh#%M5cj2oF z4Lx2ss6@;QiD{B}bG1xk?6R zx9q$`xUzp4-oO_M;3LJ*a*5| z&w=&I=mtePN&|e=O=w>3+V|oJgM-cpgKrF<;Yt&`7N)A40PmUjyvIaI3$VeJ`+gmm zj-xQF@A?wq`80MRl~jTj97PVUQ>`~}x2@C_G@%3`DpE!_ z4T^svB5P$AgXwE?w}bFA4hX1#RS7W&xwo zDKK&)jOchu%OYlq=`S&W8$;)RLuEd8Pw;a%w&6Ffe8c{iXtj057~*P!u43kASBu7RnwSzX__T}BNi`z#P}ilD!0rS# zIdM$L?)2|(S<_r2@U(?Z(5qq!HBY)jjTNU&H@5S0fcYA-Y&|81q2@5Uv*PXGW4gb2YcGTYA=G8>pzt&b; zWm=d}MF?dPL;=t>G`9Z?n@C;`C@>D!b(_nXjGbxS-z;o3tg&^Dnzs348HghcJ{(6x z5pUjFS0#K1S^1i9^LwK!^&rxUYOCB=d9~WygoB93d9-h0bn&7sK*tt@9#?<2!$%|U z3%4e0BA2RYTC0do>Ud6Cs3MP?!HBl{z%C;rrBHQWbu7twtmp*L46en5w=?mHH! z*+T9DZMVgUCbDnK`T5}W9`SP1Zqh~v8GhYX=J>K#XH68?so<~0}$ z%`CxlNUL1V>?RB%o+C6X((-FZQD2*Ja}A&txQv|MT*uK1$4>x>!-{`*qhFH)`}SBRXLwg7VDNE z@5&9rFKU0+loIfG$2TTt?HEycY0kY-kf2b{tNK@9lNK0Un z#9(n?CCV9>u<|>@%#ENwF5r2h7w80@TUV!Bwt4cZz-!=E!DJ+t<)>sZdGmeahHw8; z-Vg!U824fJJR*NrV@RxtqBqzeXg~@BWm6)2P2N@<9DAFDgGxH@cv3|sO28X+OXnUj zoK2mbc#`!A`eCYdcA^<|KR?O#f}T#8h3%_f;gp?CR~sx0j|W71%Gg=6S&DKF!|1&Y z&?ZWv8G25FrZT(Wa)|C#JjPZFO~Z(ZY}_?nCg{vH(8YfrO_&+=l;bA5aPVQmWkYts zb;+4hO2&8(RRni(VfIFrhm38#T`*IFYwCKr{zcXhMM36bwVi+@F#(C^x}5pB0XT_? zKGYM<>B0r`7+A;{Ja$vWmpojru%SFTe(6mkYd4MgA5FY|1TmRVm-Zvs#URMe3FlJK zvzuxhP`iIe1%85wNV#|2AnnN}Xgx5Tf9$WSOsh}QSafNi7=Z5A_LgUg-HZM=^>cCxatIWg5s?7#m!AMmj#syY?gOF}k0FuFpE2zsM*K^1Jp# z9HO0{xL1U+4BdsPwC{E^4`o>Dmu6QXoVOx_N_DYaZXate4<53Y-U8j@u271PV|srY zWuk2Q*p!BsmtR9)6#3i-h;uM0n1FEzp_Fw55ik}ka1DJ4tDmgx97So41&CDZHvv6d?$(htSdq?u0Mg*4gF|6Xm|at&LL zt4%ppXdbKg2)1z8)SIEdFYD~GoX++((`uewP8Tz%E_PMb=$}QA<9yjf=1f705&*a% zS$Df&*Dl71@9|!qEo)4iXd?1UrT<-ADk@Kp+{oJIz_$p^fk;N;@%Vvqn!$fp*2DQ- z_D2v9_M5rre0gzEu7^rp#j12*Sm_`mkO}p7^(dIKn#b>UKWKwiyQHx5#}@S*MLs6n%)??fzVJziMZ=KMQeA$Fwol8EO~K%$tFS&zs~-h zN=;4e-R#Bxue}S_y=@cQR2_dO7)_plFkE-JnEr<*E zz~M{E*oycG9Ol0pxC5ZWr=Bk$u>Es34jcVNZ{^~mLL}aKbzVZ9dn|vdaBaK=(EN(5 z&wzVA^tZ~Q^wIQkvvD$u#@{w?M{Ltu3F=x4dzSN#09}Xfew1$@s`_+e?G-rZ(+xU~ zMIX1ZiEtM)8zlW34x%8Fl?e*;a`OPO*rLA*o~ z0>2Zjov0HnvMY&teItSIzN~fln*|gmzVY2l%r1p!dd?4%UNDWm9IimvgysUx&bG6u z|NDOSwJit%|27XepwRwHqw5)sAm^vtSEPF2mmVDZnFjdicpQJP9?{U@(WuBq`@-OE z<4&PS71{vXMss-V`BWh9R6Y3Bq4{1TT{GVO*{pgmN|M;8^1_}SFhE6LJ3$~F=B*AG zcKb>4B8GV{i}1ndn~UZb#7Zd0*I0hn^@aC;;^59t!~zS$0MKuqfAi;8q6ETNtU{<$ z|M1iIlW*U?{^@_syRm=WIPIuEZplOfjCHaCsQJnAd~(?LI>^H}QT>H?e|i1myRk>d zvDeAwiFbOvjKB7K(3K=mGGh{N54jF{(GAiANm7*P#+1%% z>?I^L(FK407=PXi%5g4f@bQ2k&^1y=r~==araGG*OrBickrT`NAP+(L6Q%4f<`6Af zug}KgS-tTD+eBu?k>xlPB+~CfD&nb_x~4?{d+kRw$BA1hM?Bsf<{P{J6T>Xp2)5`% zkVO}AD_W_kxVy;GS?GxP-P28^?>5=W`b(2dgf)LS*+y8JlWiP(MY5ISZE9^r(~C4q zoC8n7fFOIs=MW{yp#Yw~0d&1P&@pvUJSaq@R7DIE>>yo)hC(rZi?ATo zIO(W#pLGx6coHjL<%NI=d4y6v?kJ0N5-W60N<$NoXwuNi*}(|xN0azq%5i$-GZUrV z5#)cPQi_x6ZKJp{^-!7h#muQ${e? zcN~W{5R4T$Mc9lX1$Ro;q8t?nBpWcg6fF*30P}LHBQ0v+jqn}Rx8mN!jdyaASl53O z^w+j$E^y90SlOJCFJkfuDPk(ze-(8l7WyL!q3wS~2NQGZH()0V=ub67#2~3(Dq@8gS!zHm?~s zY7(9LU|l;w-Vau#A6JvsgD$#W`bK?nH0qH~qEtez+JO|cy2DBHz=l_%=6!#k*L@Z| zes#g4lj-J;(#b!DYVAug;Ei=#q-ZkI^bXQZ2dr(U2!i6{?=Yz{TV5|1Fx7b02XKj3 z=hc};w`AlyQGIN4tyu8(bilc*lv+Bgpcxn{FAy`fN4F2)T@pW{E1Z-2;j#sXuxcTZ zvmw6|u9F=U4R%gWp6YIVam|0}4cR*0-z3+N3IDWOa1^Fr9MA%M2dLf!-sGx-lM~=D z>NQKaKCY{}^pL3S%tk@6iJs4kaa^M`?zsbK`~R?9gGGW#!AHVAP9jSR0GR>W<3B~Y zhZK68C9JVeS3{o)VgyhGIi&9J3%NHOZGe* zUD4~5!R+bX5CboV2v>icJ2ol+>W|6``jM)WE>$vFtQ0#kyEimt?H4Pjs*&H$RJiAM zC_e_BVS#&Y!9XkWcls)Tc?UX2CFY|GT03H>4F~(x_9nZKrOs=$B1lfW@75pS*ISL` zd3UO@2?2OV$EHf4OK!FrmnA?iCW;_>4dD#Uw@2=S$Fb|k6g_{JiDxQZ>jOIiPp_=+)*6$mND>~u6GNS5#{Z zAJrhfq7f3@q=5gE*dYsVAf}2RbBQOm+h-KmYxz~~7YKc;RLSZpULq8i7+rD(9H{(o zMq)pB-d6MJ0>pp(v*~JjR&5lI@XLjQj*xu}#&xU9LlvaIeDp^d2Bu>ib7Y~Lmu4gc zT*2~9@(r*F_cCx#I|BlEx2lE(hz`O@_xBEg$@tZ|L!vfcFPhC&$Vof*SP{9ho_LvE zUiN}m;Un6UW6cT(I(jo*Z_soSEwJT2M#*N*;vJ__q4a+R7kw+piYB44LjshhCX+hh z--)Vsz8p#Rnk}W+g@f%kh+Mp9i_~ok*4MajK*>Si)uh~*zN_H5G6KUlg88Mt$cso* zBPO&rC1D}f&Iq82kuKDI=EddE5Vh5CJ)zL2QT9%ML*-qHSYZk&dXYTniC*B;U#?(B z=cP{EFi(Gr*jQK_<5t@PM z#|m_}u>VHQSCf(%n?Bnr9VI~ zoEiBz*v+BeCO-JslVKxZisi;s5_MNC(t{20!qvyN_0W6JaCrI~t6Oc#^=7%8E2_Sn zR*Swrha$H6Ublk#dc-$eEmwVNr7Pj~*km{P6WFoeaInn}N?w@#2x4sRR~(*kd@H;2 zbo74;pe{o!CdQe}(XWEh3z$TtA(^RQ1#z#9I!Fz5GJ1~rKUN@>oc@ttq>APINGxHZ zt2RgCizpPKWo&FX+u?)ntQ`Ml7VX6r@_KDp@Bj*WY5QgYWQD7waoOs#>0F&mQ8Mr? z@>sGPmUdZBso#02xWE~nm^xcb*Wh1+HrSIB;o&|x$rOcR zXi>c>r=}N)Qk+2Ciy1OF$DmIr`v&iV1nz`e3Z#ZkZqf72iS6degi-&F5WPpE( z-j2lW)hj0`Bp~|ickuTM_|q6b@J5qQhnA4zj;4@LX%}iWS_u}?#z9Qspxm!Psu+o> z@Tr5k&TU8_dgwVyLUC^r(E8~*Wqg}@S17Xaaf%<6VD>}5hdf?Y}3?VmgZ@O`>Bt{tOcrj7y=b(B*5WpEAE?h6yN_f3uqcMOHjDUA5 z5&(9ORy2n>|K7pV-VI)BHtCSdX>#DSqPe5zMfO689miCofbgm7?2vzwvU(B|3i~*s zWUYgOv$hV8vC9Pv46!|6bR+Q`p+)YG%$xL+BK=yoIvDBx4-0k4{pq{E{Kfxo_|W0V zi8yfKA$6m#nj1Y=b63oS)$aqiOyg0ScF%wooFgSjSsfqs|1GrUu?c-NKyYxnmOP{3 zniHJUa2~}Et4N0pTf8T{b(;<-3Nj~e1SNd6j&M><<=)ev)7G2i9TGFPeU z1TTLUL){AQ6CXq z&`G>auzK-khW!^Q!J}D7@(L1mBF4# zdK?b|al%f={`8viPsZ--S~60`!Tg${61{K|nznnq8g2f@Vh{xEPUrJSHyb>v%HPSz zpl=x_Vi8~A#i?0!`}ur6zyLs5$-0lReUNsg7X5rXCj*+-d-g2b=Y9xs%7KG2=?h$6 zflKOuV8q$oBIkcxBT!P+@_Na}@DCf(^TB{VU7THl=#EH!15QL(R~wx}pwBPGUx3_D zQwoHe0Wk&3KG+bQ)Cv>n9$5AKIe(pU|4a&3;kU7FO;K(g)oQf`^9iMRAfWBosmRV_ zbI5nrRSJ(;;6pt~s8S4=e98^Ip z(Q@>G-|qqD9~{eS^1f0!?zm|PIx^~Rio76Tq0{pV>7q}o>V=Ynoe`R3*3t3sTq!zz z{d?26<;8#9bP<8k5!zsf|4Lgz=2kb%Ho1Ry$dXG)K5XiVJ<&#VT`tDo%zPcXsnrYy zH6UV=p;=ynrj9n});-Z7>rdG82iGS1sQxnjd!=s7Q+3zX!qvt3ak{!a+pGQRNE%R2 z>+QTe1N|6T7SMwX&MdB~hshA=`6{SXjs@PK*Rp>x9zq-e_GZACIXnV$QF|%gT$cQF z3ge>j;+ESOhc9b@(-9zz_O=G!ExM{Hd!JX*y;Q+NHb#OT#=~rPF>t+Hb3Wf{xu`1* z;&?%jK2Yi{vNKP~*saktwS|0|ZPwN5x>}cT3vDi!O1Z~$2}H+J!a{YfmSHtrbDF)j zjBkHFr2J`j^w~l4ROQpQXCkFd+aB?2mpbix!m{0D)4sDMyHN*iRL;66rlIBf(X7_O zV%=t|J`VDmJxwcy9M+&|B{=oCHCrnJvNKK)rK?b-z<0(?G#YjhNfaD6P;A(NkJ>X% z5wCreS9~{-udE~sN>74-3<=N)q}%u;W;%aAl?lzL{wx)9U=72B^DNmkwacY1TjI-6 z^F-x|)0eTr9sZt!p3~Fl#$R+6oF76`kvLweb&`MfJOR$ftS%-g8 z!#kKdhMCBll%_B4;ZUZK3G*yE{5+li0*P;-pbBYlN@p!l&oyBv79U4o^6G%Hl5!f% zA>Px6aWBMGBOo1evTxASo2?+Nb*5KUkuC_kSfmxJsMK2w+S{(J1`SXB`G5ZFzh5#W zANdrd7-EOU!Uy=T<~W~kjTlj5;bDJXgAXSs!;Ga|wb}0>gi8IWNa^In$TVF06pCgV z41vf>Lq%Y8YHPrQ?-uW;>uS1CR3`HzN&1FVeLyuvuZm*pEM%G3Bs{&C#?NOL z*Y02iJ9jw+fzE^tnv_c$26^JjMgK=c3p)^LHff9x#D;7N5Jm}ST6Z#M*p+{y)SAS_ zsFQF^7p8UcBIHNR$~jNOEF9jc z5X;i{Y>;09C~&Vw0Pe4CbS$P@& None: @@ -160,7 +156,7 @@ index 48343f3d694b8fe2e5ea85e3e173c05f23b01c0d..7ed0b0344008374c9e00199730601230 return B12xWarmupUnit( name="MXFP8", -@@ -137,6 +154,8 @@ class B12xMxfp8LinearKernel(Mxfp8LinearKernel): +@@ -137,6 +154,8 @@ int(packed_weight.in_features), int(packed_weight.padded_in_features), int(packed_weight.out_features), @@ -169,11 +165,9 @@ index 48343f3d694b8fe2e5ea85e3e173c05f23b01c0d..7ed0b0344008374c9e00199730601230 output_dtype, ), compile=compile, -diff --git a/vllm/model_executor/kernels/linear/nvfp4/b12x.py b/vllm/model_executor/kernels/linear/nvfp4/b12x.py -index 4ff531dd48be9ec7808872847d9665e1b893d7a6..aee238b3f30fb5a871f3b212b58dcd37916c0fed 100644 --- a/vllm/model_executor/kernels/linear/nvfp4/b12x.py +++ b/vllm/model_executor/kernels/linear/nvfp4/b12x.py -@@ -8,7 +8,7 @@ import torch +@@ -8,7 +8,7 @@ from vllm._custom_ops import scaled_fp4_quant from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform @@ -182,7 +176,7 @@ index 4ff531dd48be9ec7808872847d9665e1b893d7a6..aee238b3f30fb5a871f3b212b58dcd37 from vllm.utils.b12x import ( get_b12x_blockscaled as _import_b12x_blockscaled, ) -@@ -18,30 +18,40 @@ from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig +@@ -18,30 +18,40 @@ def _apply_b12x_nvfp4_linear( @@ -232,7 +226,7 @@ index 4ff531dd48be9ec7808872847d9665e1b893d7a6..aee238b3f30fb5a871f3b212b58dcd37 out_dtype=x.dtype, ) if bias is not None: -@@ -66,6 +76,11 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel): +@@ -66,6 +76,11 @@ return False, "Install the B12X backend with `pip install vllm[b12x]`" if not blockscaled.is_supported(): return False, "b12x native NVFP4 GEMM is not supported" @@ -244,10 +238,11 @@ index 4ff531dd48be9ec7808872847d9665e1b893d7a6..aee238b3f30fb5a871f3b212b58dcd37 return True, None @classmethod -@@ -81,6 +96,21 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel): +@@ -80,6 +95,21 @@ + layer, "weight_scale", intrinsics.swizzle_block_scale(layer.weight_scale.data), - ) ++ ) + blockscaled = _import_b12x_blockscaled() + assert blockscaled is not None + layer.b12x_nvfp4_packed_weight = blockscaled.pack_weight( @@ -262,11 +257,10 @@ index 4ff531dd48be9ec7808872847d9665e1b893d7a6..aee238b3f30fb5a871f3b212b58dcd37 + current_platform.is_device_capability_family(120) + and (packed_k * 2) % 128 == 0 + and n % 8 == 0 -+ ) + ) layer.b12x_warmup_provider = self - def get_b12x_warmup_unit( -@@ -100,11 +130,8 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel): +@@ -100,11 +130,8 @@ (tokens, k), dtype=output_dtype, device=weight.device ) _apply_b12x_nvfp4_linear( @@ -279,7 +273,7 @@ index 4ff531dd48be9ec7808872847d9665e1b893d7a6..aee238b3f30fb5a871f3b212b58dcd37 None, ) -@@ -117,6 +144,8 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel): +@@ -117,6 +144,8 @@ k, weight.dtype, weight_scale.dtype, @@ -288,7 +282,7 @@ index 4ff531dd48be9ec7808872847d9665e1b893d7a6..aee238b3f30fb5a871f3b212b58dcd37 output_dtype, ), compile=compile, -@@ -129,11 +158,8 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel): +@@ -129,11 +158,8 @@ bias: torch.Tensor | None = None, ) -> torch.Tensor: return _apply_b12x_nvfp4_linear( @@ -301,11 +295,9 @@ index 4ff531dd48be9ec7808872847d9665e1b893d7a6..aee238b3f30fb5a871f3b212b58dcd37 bias, ) -diff --git a/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py -index aacfb1dc59bac0507c4ad2b743b3be1137ad054e..9fe6ab86c26d4685fb07d9407ea67fad1e01e872 100644 --- a/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py -@@ -991,8 +991,7 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): +@@ -991,8 +991,7 @@ """Execute B12X KDA after the convolution projection. Args: @@ -315,7 +307,7 @@ index aacfb1dc59bac0507c4ad2b743b3be1137ad054e..9fe6ab86c26d4685fb07d9407ea67fad mixed_qkv: Live packed query, key, and value projection. raw_g: Live unactivated forget gate. raw_beta: Live unactivated update gate. -@@ -1032,17 +1031,16 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): +@@ -1032,17 +1031,16 @@ cache = forward_context.additional_kwargs.setdefault( "b12x_kda_metadata_tensors", {} ) @@ -336,7 +328,7 @@ index aacfb1dc59bac0507c4ad2b743b3be1137ad054e..9fe6ab86c26d4685fb07d9407ea67fad if num_accepted_tokens is None: accepted_tokens = self._b12x_kda_num_accepted_tokens[:num_requests] accepted_tokens.fill_(1) -@@ -1055,7 +1053,6 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): +@@ -1055,7 +1053,6 @@ bound_metadata = ( query_start_loc, accepted_tokens, @@ -344,7 +336,7 @@ index aacfb1dc59bac0507c4ad2b743b3be1137ad054e..9fe6ab86c26d4685fb07d9407ea67fad self._b12x_kda_num_seqs, self._b12x_kda_num_tokens, ) -@@ -1063,7 +1060,6 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): +@@ -1063,7 +1060,6 @@ ( query_start_loc, accepted_tokens, @@ -352,7 +344,7 @@ index aacfb1dc59bac0507c4ad2b743b3be1137ad054e..9fe6ab86c26d4685fb07d9407ea67fad num_seqs, num_tokens_tensor, ) = bound_metadata -@@ -1081,7 +1077,7 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): +@@ -1081,7 +1077,7 @@ recurrent_state=self.kv_cache[1], query_start_loc=query_start_loc, num_accepted_tokens=accepted_tokens, @@ -361,121 +353,39 @@ index aacfb1dc59bac0507c4ad2b743b3be1137ad054e..9fe6ab86c26d4685fb07d9407ea67fad num_seqs=num_seqs, num_tokens=num_tokens_tensor, output=output, -diff --git a/vllm/model_executor/layers/quantization/online/nvfp4.py b/vllm/model_executor/layers/quantization/online/nvfp4.py -index de920db901aa695d2a0eb3798c4ae67d64bd181c..3d36f872b0fb87482cdc48051ff7d3adeab959d7 100644 --- a/vllm/model_executor/layers/quantization/online/nvfp4.py +++ b/vllm/model_executor/layers/quantization/online/nvfp4.py -@@ -1,192 +1,261 @@ --# SPDX-License-Identifier: Apache-2.0 --# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -- --import torch -+# SPDX-License-Identifier: Apache-2.0 -+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -+ -+import torch +@@ -5,6 +5,10 @@ from torch.nn import Module from vllm._custom_ops import scaled_fp4_quant --from vllm.model_executor.layers.fused_moe import RoutedExperts --from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig --from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( -- convert_to_nvfp4_moe_kernel_format, -- make_nvfp4_moe_kernel, -- make_nvfp4_moe_quant_config, -- select_nvfp4_moe_backend, +from vllm.model_executor.kernels.linear.nvfp4.b12x import ( + B12xNvFp4LinearKernel, - ) -+from vllm.model_executor.kernels.linear.nvfp4.base import NvFp4LinearLayerConfig -+from vllm.model_executor.layers.fused_moe import RoutedExperts -+from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig -+from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( -+ convert_to_nvfp4_moe_kernel_format, -+ make_nvfp4_moe_kernel, -+ make_nvfp4_moe_quant_config, -+ select_nvfp4_moe_backend, +) ++from vllm.model_executor.kernels.linear.nvfp4.base import NvFp4LinearLayerConfig + from vllm.model_executor.layers.fused_moe import RoutedExperts + from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig + from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( +@@ -13,6 +17,7 @@ + make_nvfp4_moe_quant_config, + select_nvfp4_moe_backend, + ) +from vllm.model_executor.layers.quantization.online.fp8 import _Fp8OnlineLinearBase from vllm.model_executor.layers.quantization.online.moe_base import ( OnlineMoEMethodBase, ) --from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( -- FLOAT4_E2M1_MAX, --) --from vllm.model_executor.layers.quantization.utils.quant_utils import ( -- amax_for_moe_weight_quant, -- kNvfp4Dynamic, -- kNvfp4Static, -- weight_amax, --) --from vllm.model_executor.utils import replace_parameter -+from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( -+ FLOAT4_E2M1_MAX, -+) -+from vllm.model_executor.layers.quantization.utils.quant_utils import ( -+ amax_for_moe_weight_quant, -+ kNvfp4Dynamic, -+ kNvfp4Static, -+ weight_amax, -+) -+from vllm.model_executor.utils import replace_parameter +@@ -27,8 +32,72 @@ + ) + from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform -- +from vllm.utils.b12x import get_b12x_blockscaled -+ - FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max - --def _quantize_moe_weight_to_nvfp4( -- weight: torch.Tensor, -- moe_tp_size: int = 1, --) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: -- """Quantize stacked MoE expert weights ``(E, N, K)`` to NVFP4. -- -- One FP32 global scale per expert plus per-block (group-16) FP8 scales, -- matching the ModelOpt NVFP4 checkpoint layout. Returns packed FP4 weights -- ``(E, N, K // 2)``, block scales ``(E, N, K // 16)``, and the per-expert -- global scale ``(E,)`` stored as ``amax / (fp4_max * fp8_max)``. -- """ -- assert weight.dim() == 3, f"expected 3D expert weights, got {weight.shape}" -- k = weight.shape[-1] -- assert k % 16 == 0, f"last dim must be a multiple of 16, got {k}" -- -- amax = weight_amax(weight.flatten(1), dim=-1).to(torch.float32) -- amax = amax_for_moe_weight_quant(amax, moe_tp_size).clamp_min(1e-8) -- global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax -- weight_scale_2 = (1.0 / global_scale).to(torch.float32) + FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max ++ ++ +class Nvfp4OnlineLinearMethod(_Fp8OnlineLinearBase): + """Load a BF16 linear weight as NVFP4 for the proposal head.""" - -- # Keep the original BF16/FP16 values as the quantizer input. Folding each -- # expert's FP32 global scale into the weight would add a BF16/FP16 rounding -- # before the group-16 scale and E2M1 values are selected. -- weight = weight.contiguous() -- quantized_experts = [ -- scaled_fp4_quant( -- expert_weight, -- expert_scale, -- is_sf_swizzled_layout=False, -- ) -- for expert_weight, expert_scale in zip( -- weight, -- global_scale, -- strict=True, -- ) -- ] -- qweight = torch.stack([quantized for quantized, _ in quantized_experts]) -- block_scale = torch.stack([block_scale for _, block_scale in quantized_experts]) -- return ( -- qweight, -- block_scale, -- weight_scale_2, -- ) -- -- --class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase): -- """Online NVFP4 MoE quantization with per-token activation scales. ++ + def __init__(self, *, use_a16: bool = False): + super().__init__() + supported, reason = B12xNvFp4LinearKernel.is_supported() @@ -485,45 +395,10 @@ index de920db901aa695d2a0eb3798c4ae67d64bd181c..3d36f872b0fb87482cdc48051ff7d3ad + self.use_a16 = use_a16 + if use_a16 and self.input_dtype != torch.bfloat16: + raise ValueError("A16 proposal heads require BF16 activations") - -- Quantizes fp16/bf16 expert weights to NVFP4 at load time; the FlashInfer -- TRTLLM kernel computes per-token activation scales at runtime. Blackwell -- (SM100) only. -- """ -- -- def __init__( -- self, -- *, -- layer: torch.nn.Module, -- ): -- if not current_platform.is_device_capability_family(100): -- raise ValueError( -- "nvfp4_per_token online quantization requires a Blackwell (SM100) GPU." -- ) -- super().__init__(layer.moe_config) -- self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend( -- config=self.moe, -- weight_key=kNvfp4Static, -- activation_key=kNvfp4Dynamic, -- ) -- -- def process_weights_after_loading(self, layer: Module) -> None: ++ + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - if getattr(layer, "_already_called_process_weights_after_loading", False): - return -- -- self._quantize_weights(layer) -- self._setup_kernel(layer) -- -- layer._already_called_process_weights_after_loading = True -- -- def _quantize_weights(self, layer: Module) -> None: -- moe_tp_size = self.moe.tp_size -- w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4( -- layer.w13_weight, moe_tp_size -- ) -- w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4( -- layer.w2_weight, moe_tp_size ++ if getattr(layer, "_already_called_process_weights_after_loading", False): ++ return + weight = layer.weight.contiguous() + if weight.shape[1] % 16: + raise ValueError("Online NVFP4 proposal head requires K divisible by 16") @@ -531,7 +406,7 @@ index de920db901aa695d2a0eb3798c4ae67d64bd181c..3d36f872b0fb87482cdc48051ff7d3ad + global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax + packed, scales = scaled_fp4_quant( + weight, global_scale, is_sf_swizzled_layout=False - ) ++ ) + replace_parameter(layer, "weight", packed) + replace_parameter(layer, "weight_scale", scales) + replace_parameter(layer, "weight_global_scale", global_scale.reciprocal()) @@ -540,42 +415,7 @@ index de920db901aa695d2a0eb3798c4ae67d64bd181c..3d36f872b0fb87482cdc48051ff7d3ad + self.kernel.process_weights_after_loading(layer) + layer.b12x_activation_mode = "a16" if self.use_a16 else "quantized" + layer._already_called_process_weights_after_loading = True - -- replace_parameter(layer, "w13_weight", w13) -- replace_parameter(layer, "w13_weight_scale", w13_scale) -- replace_parameter(layer, "w13_weight_scale_2", w13_scale_2) -- replace_parameter(layer, "w2_weight", w2) -- replace_parameter(layer, "w2_weight_scale", w2_scale) -- replace_parameter(layer, "w2_weight_scale_2", w2_scale_2) -- -- # Neutral (1.0) activation global scales: the kernel derives per-token -- # scales at runtime, so the output scalars reduce to the weight scales. -- ones = torch.ones(layer.num_experts, device=w13.device, dtype=torch.float32) -- replace_parameter(layer, "w13_input_scale", ones) -- replace_parameter(layer, "w2_input_scale", ones.clone()) -- -- def _setup_kernel(self, layer: RoutedExperts) -> None: -- ( -- w13, -- w13_scale, -- w13_scale_2, -- a13_scale, -- w2, -- w2_scale, -- w2_scale_2, -- a2_scale, -- ) = convert_to_nvfp4_moe_kernel_format( -- nvfp4_backend=self.nvfp4_backend, -- layer=layer, -- w13=layer.w13_weight, -- w13_scale=layer.w13_weight_scale, -- w13_scale_2=layer.w13_weight_scale_2, -- a13_scale=layer.w13_input_scale, -- w2=layer.w2_weight, -- w2_scale=layer.w2_weight_scale, -- w2_scale_2=layer.w2_weight_scale_2, -- a2_scale=layer.w2_input_scale, -- is_act_and_mul=self.moe.is_act_and_mul, ++ + def apply( + self, + layer: torch.nn.Module, @@ -590,42 +430,7 @@ index de920db901aa695d2a0eb3798c4ae67d64bd181c..3d36f872b0fb87482cdc48051ff7d3ad + x.reshape(-1, x.shape[-1]), + input_scale.reciprocal(), + is_sf_swizzled_layout=True, - ) -- -- replace_parameter(layer, "w13_weight", w13) -- replace_parameter(layer, "w13_weight_scale", w13_scale) -- replace_parameter(layer, "w13_weight_scale_2", w13_scale_2) -- replace_parameter(layer, "w13_input_scale", a13_scale) -- replace_parameter(layer, "w2_weight", w2) -- replace_parameter(layer, "w2_weight_scale", w2_scale) -- replace_parameter(layer, "w2_weight_scale_2", w2_scale_2) -- replace_parameter(layer, "w2_input_scale", a2_scale) -- -- if self.moe_kernel is None: -- self.moe_quant_config = self.get_fused_moe_quant_config(layer) -- assert self.experts_cls is not None -- self.moe_kernel = make_nvfp4_moe_kernel( -- moe_quant_config=self.moe_quant_config, -- moe_config=self.moe, -- experts_cls=self.experts_cls, -- backend=self.nvfp4_backend, -- routing_tables=layer._expert_routing_tables(), -- per_token_activation=True, -- ) -- -- self.moe_kernel.fused_experts.process_weights_after_loading(layer) -- -- def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: -- return make_nvfp4_moe_quant_config( -- backend=self.nvfp4_backend, -- w13_scale=layer.w13_weight_scale, -- w2_scale=layer.w2_weight_scale, -- w13_scale_2=layer.w13_weight_scale_2, -- w2_scale_2=layer.w2_weight_scale_2, -- a13_scale=layer.w13_input_scale, -- a2_scale=layer.w2_input_scale, -- swiglu_limit=getattr(layer, "swiglu_limit", None), -- layer=layer, ++ ) + blockscaled = get_b12x_blockscaled() + assert blockscaled is not None + output = blockscaled.mm_nvfp4( @@ -635,2443 +440,95 @@ index de920db901aa695d2a0eb3798c4ae67d64bd181c..3d36f872b0fb87482cdc48051ff7d3ad + layer.weight_scale, + input_scale * layer.weight_global_scale, + out_dtype=x.dtype, - ) ++ ) + if bias is not None: + output = output + bias + return output.view(*x.shape[:-1], layer.weight.shape[0]) -+ -+ -+def _quantize_moe_weight_to_nvfp4( -+ weight: torch.Tensor, -+ moe_tp_size: int = 1, -+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: -+ """Quantize stacked MoE expert weights ``(E, N, K)`` to NVFP4. -+ -+ One FP32 global scale per expert plus per-block (group-16) FP8 scales, -+ matching the ModelOpt NVFP4 checkpoint layout. Returns packed FP4 weights -+ ``(E, N, K // 2)``, block scales ``(E, N, K // 16)``, and the per-expert -+ global scale ``(E,)`` stored as ``amax / (fp4_max * fp8_max)``. -+ """ -+ assert weight.dim() == 3, f"expected 3D expert weights, got {weight.shape}" -+ k = weight.shape[-1] -+ assert k % 16 == 0, f"last dim must be a multiple of 16, got {k}" -+ -+ amax = weight_amax(weight.flatten(1), dim=-1).to(torch.float32) -+ amax = amax_for_moe_weight_quant(amax, moe_tp_size).clamp_min(1e-8) -+ global_scale = (FLOAT4_E2M1_MAX * FLOAT8_E4M3_MAX) / amax -+ weight_scale_2 = (1.0 / global_scale).to(torch.float32) -+ -+ # Keep the original BF16/FP16 values as the quantizer input. Folding each -+ # expert's FP32 global scale into the weight would add a BF16/FP16 rounding -+ # before the group-16 scale and E2M1 values are selected. -+ weight = weight.contiguous() -+ quantized_experts = [ -+ scaled_fp4_quant( -+ expert_weight, -+ expert_scale, -+ is_sf_swizzled_layout=False, -+ ) -+ for expert_weight, expert_scale in zip( -+ weight, -+ global_scale, -+ strict=True, -+ ) -+ ] -+ qweight = torch.stack([quantized for quantized, _ in quantized_experts]) -+ block_scale = torch.stack([block_scale for _, block_scale in quantized_experts]) -+ return ( -+ qweight, -+ block_scale, -+ weight_scale_2, -+ ) -+ -+ -+class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase): -+ """Online NVFP4 MoE quantization with per-token activation scales. -+ -+ Quantizes fp16/bf16 expert weights to NVFP4 at load time; the FlashInfer -+ TRTLLM kernel computes per-token activation scales at runtime. Blackwell -+ (SM100) only. -+ """ -+ -+ def __init__( -+ self, -+ *, -+ layer: torch.nn.Module, -+ ): -+ if not current_platform.is_device_capability_family(100): -+ raise ValueError( -+ "nvfp4_per_token online quantization requires a Blackwell (SM100) GPU." -+ ) -+ super().__init__(layer.moe_config) -+ self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend( -+ config=self.moe, -+ weight_key=kNvfp4Static, -+ activation_key=kNvfp4Dynamic, -+ ) -+ -+ def process_weights_after_loading(self, layer: Module) -> None: -+ if getattr(layer, "_already_called_process_weights_after_loading", False): -+ return -+ -+ self._quantize_weights(layer) -+ self._setup_kernel(layer) -+ -+ layer._already_called_process_weights_after_loading = True -+ -+ def _quantize_weights(self, layer: Module) -> None: -+ moe_tp_size = self.moe.tp_size -+ w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4( -+ layer.w13_weight, moe_tp_size -+ ) -+ w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4( -+ layer.w2_weight, moe_tp_size -+ ) -+ -+ replace_parameter(layer, "w13_weight", w13) -+ replace_parameter(layer, "w13_weight_scale", w13_scale) -+ replace_parameter(layer, "w13_weight_scale_2", w13_scale_2) -+ replace_parameter(layer, "w2_weight", w2) -+ replace_parameter(layer, "w2_weight_scale", w2_scale) -+ replace_parameter(layer, "w2_weight_scale_2", w2_scale_2) -+ -+ # Neutral (1.0) activation global scales: the kernel derives per-token -+ # scales at runtime, so the output scalars reduce to the weight scales. -+ ones = torch.ones(layer.num_experts, device=w13.device, dtype=torch.float32) -+ replace_parameter(layer, "w13_input_scale", ones) -+ replace_parameter(layer, "w2_input_scale", ones.clone()) -+ -+ def _setup_kernel(self, layer: RoutedExperts) -> None: -+ ( -+ w13, -+ w13_scale, -+ w13_scale_2, -+ a13_scale, -+ w2, -+ w2_scale, -+ w2_scale_2, -+ a2_scale, -+ ) = convert_to_nvfp4_moe_kernel_format( -+ nvfp4_backend=self.nvfp4_backend, -+ layer=layer, -+ w13=layer.w13_weight, -+ w13_scale=layer.w13_weight_scale, -+ w13_scale_2=layer.w13_weight_scale_2, -+ a13_scale=layer.w13_input_scale, -+ w2=layer.w2_weight, -+ w2_scale=layer.w2_weight_scale, -+ w2_scale_2=layer.w2_weight_scale_2, -+ a2_scale=layer.w2_input_scale, -+ is_act_and_mul=self.moe.is_act_and_mul, -+ ) -+ -+ replace_parameter(layer, "w13_weight", w13) -+ replace_parameter(layer, "w13_weight_scale", w13_scale) -+ replace_parameter(layer, "w13_weight_scale_2", w13_scale_2) -+ replace_parameter(layer, "w13_input_scale", a13_scale) -+ replace_parameter(layer, "w2_weight", w2) -+ replace_parameter(layer, "w2_weight_scale", w2_scale) -+ replace_parameter(layer, "w2_weight_scale_2", w2_scale_2) -+ replace_parameter(layer, "w2_input_scale", a2_scale) -+ -+ if self.moe_kernel is None: -+ self.moe_quant_config = self.get_fused_moe_quant_config(layer) -+ assert self.experts_cls is not None -+ self.moe_kernel = make_nvfp4_moe_kernel( -+ moe_quant_config=self.moe_quant_config, -+ moe_config=self.moe, -+ experts_cls=self.experts_cls, -+ backend=self.nvfp4_backend, -+ routing_tables=layer._expert_routing_tables(), -+ per_token_activation=True, -+ ) -+ -+ self.moe_kernel.fused_experts.process_weights_after_loading(layer) -+ -+ def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: -+ return make_nvfp4_moe_quant_config( -+ backend=self.nvfp4_backend, -+ w13_scale=layer.w13_weight_scale, -+ w2_scale=layer.w2_weight_scale, -+ w13_scale_2=layer.w13_weight_scale_2, -+ w2_scale_2=layer.w2_weight_scale_2, -+ a13_scale=layer.w13_input_scale, -+ a2_scale=layer.w2_input_scale, -+ swiglu_limit=getattr(layer, "swiglu_limit", None), -+ layer=layer, -+ ) -diff --git a/vllm/model_executor/layers/vocab_parallel_embedding.py b/vllm/model_executor/layers/vocab_parallel_embedding.py -index 3057270c62185e767cfd48f63593c2705a1b1b79..2adf622d9372d36013e16ba591738ecbc78d77c0 100644 + + + def _quantize_moe_weight_to_nvfp4( --- a/vllm/model_executor/layers/vocab_parallel_embedding.py +++ b/vllm/model_executor/layers/vocab_parallel_embedding.py -@@ -1,581 +1,604 @@ --# SPDX-License-Identifier: Apache-2.0 --# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -- -+# SPDX-License-Identifier: Apache-2.0 -+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -+ +@@ -3,6 +3,7 @@ + from collections.abc import Sequence from dataclasses import dataclass -- --import torch --import torch.nn.functional as F --from torch.nn.parameter import Parameter -- --import vllm.envs as envs --from vllm.distributed import ( -- divide, -- get_tensor_model_parallel_rank, -- get_tensor_model_parallel_world_size, -- tensor_model_parallel_all_reduce, --) --from vllm.model_executor.custom_op import PluggableLayer --from vllm.model_executor.determinism.batch_invariant import ( -- linear_batch_invariant, --) --from vllm.model_executor.layers.quantization.base_config import ( -- QuantizationConfig, -- QuantizeMethodBase, -- method_has_implemented_embedding, --) --from vllm.model_executor.layers.utils import dispatch_unquantized_gemm --from vllm.model_executor.parameter import BasevLLMParameter --from vllm.model_executor.utils import set_weight_attrs --from vllm.platforms import current_platform -- --DEFAULT_VOCAB_PADDING_SIZE = 64 -- -- --class UnquantizedEmbeddingMethod(QuantizeMethodBase): -- """Unquantized method for embeddings.""" -- -- def create_weights( -- self, -- layer: torch.nn.Module, -- input_size_per_partition: int, -- output_partition_sizes: list[int], -- input_size: int, -- output_size: int, -- params_dtype: torch.dtype, -- **extra_weight_attrs, -- ): -- """Create weights for embedding layer.""" -- weight = Parameter( -- torch.empty( -- sum(output_partition_sizes), -- input_size_per_partition, -- dtype=params_dtype, -- ), -- requires_grad=False, -- ) -- set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) -- layer.register_parameter("weight", weight) -- set_weight_attrs(weight, extra_weight_attrs) -- -- def process_weights_after_loading(self, layer: torch.nn.Module) -> None: -- if current_platform.is_cpu(): -- from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm -- -- dispatch_cpu_unquantized_gemm(layer, remove_weight=False) -- -- def apply( -- self, -- layer: torch.nn.Module, -- x: torch.Tensor, -- bias: torch.Tensor | None = None, -- ) -> torch.Tensor: -- if envs.VLLM_BATCH_INVARIANT and current_platform.is_cuda_alike(): -- return linear_batch_invariant(x, layer.weight, bias) -- return dispatch_unquantized_gemm()(layer, x, layer.weight, bias) -- -- def embedding(self, layer: torch.nn.Module, input_: torch.Tensor) -> torch.Tensor: -- return F.embedding(input_, layer.weight) -- -- def tie_weights( -- self, layer: torch.nn.Module, embed_tokens: "VocabParallelEmbedding" -- ): -- layer.weight = embed_tokens.weight -- return layer -- -- --def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int: -- """Pad the vocab size to the given value.""" -- return ((vocab_size + pad_to - 1) // pad_to) * pad_to -- -- --def vocab_range_from_per_partition_vocab_size( -- per_partition_vocab_size: int, rank: int, offset: int = 0 --) -> Sequence[int]: -- index_f = rank * per_partition_vocab_size -- index_l = index_f + per_partition_vocab_size -- return index_f + offset, index_l + offset -- -- --def vocab_range_from_global_vocab_size( -- global_vocab_size: int, rank: int, world_size: int, offset: int = 0 --) -> Sequence[int]: -- per_partition_vocab_size = divide(global_vocab_size, world_size) -- return vocab_range_from_per_partition_vocab_size( -- per_partition_vocab_size, rank, offset=offset -- ) -- -- --@dataclass --class VocabParallelEmbeddingShardIndices: -- """Indices for a shard of a vocab parallel embedding.""" -- -- padded_org_vocab_start_index: int -- padded_org_vocab_end_index: int -- padded_added_vocab_start_index: int -- padded_added_vocab_end_index: int -- -- org_vocab_start_index: int -- org_vocab_end_index: int -- added_vocab_start_index: int -- added_vocab_end_index: int -- -- @property -- def num_org_elements(self) -> int: -- return self.org_vocab_end_index - self.org_vocab_start_index -- -- @property -- def num_added_elements(self) -> int: -- return self.added_vocab_end_index - self.added_vocab_start_index -- -- @property -- def num_org_elements_padded(self) -> int: -- return self.padded_org_vocab_end_index - self.padded_org_vocab_start_index -- -- @property -- def num_added_elements_padded(self) -> int: -- return self.padded_added_vocab_end_index - self.padded_added_vocab_start_index -- -- @property -- def num_org_vocab_padding(self) -> int: -- return self.num_org_elements_padded - self.num_org_elements -- -- @property -- def num_added_vocab_padding(self) -> int: -- return self.num_added_elements_padded - self.num_added_elements -- -- @property -- def num_elements_padded(self) -> int: -- return self.num_org_elements_padded + self.num_added_elements_padded -- -- def __post_init__(self): -- # sanity checks -- assert self.padded_org_vocab_start_index <= self.padded_org_vocab_end_index -- assert self.padded_added_vocab_start_index <= self.padded_added_vocab_end_index -- -- assert self.org_vocab_start_index <= self.org_vocab_end_index -- assert self.added_vocab_start_index <= self.added_vocab_end_index -- -- assert self.org_vocab_start_index <= self.padded_org_vocab_start_index -- assert self.added_vocab_start_index <= self.padded_added_vocab_start_index -- assert self.org_vocab_end_index <= self.padded_org_vocab_end_index -- assert self.added_vocab_end_index <= self.padded_added_vocab_end_index -- -- assert self.num_org_elements <= self.num_org_elements_padded -- assert self.num_added_elements <= self.num_added_elements_padded -- -- --@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend) --def get_masked_input_and_mask( -- input_: torch.Tensor, -- org_vocab_start_index: int, -- org_vocab_end_index: int, -- num_org_vocab_padding: int, -- added_vocab_start_index: int, -- added_vocab_end_index: int, --) -> tuple[torch.Tensor, torch.Tensor]: -- # torch.compile will fuse all of the pointwise ops below -- # into a single kernel, making it very fast -- org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ < org_vocab_end_index) -- added_vocab_mask = (input_ >= added_vocab_start_index) & ( -- input_ < added_vocab_end_index -- ) -- added_offset = ( -- added_vocab_start_index -- - (org_vocab_end_index - org_vocab_start_index) -- - num_org_vocab_padding -- ) -- valid_offset = (org_vocab_start_index * org_vocab_mask) + ( -- added_offset * added_vocab_mask -- ) -- vocab_mask = org_vocab_mask | added_vocab_mask -- input_ = vocab_mask * (input_ - valid_offset) -- return input_, ~vocab_mask -- -- --# --8<-- [start:vocab_parallel_embedding] --@PluggableLayer.register("vocab_parallel_embedding") --class VocabParallelEmbedding(PluggableLayer): -- """Embedding parallelized in the vocabulary dimension. -- -- Adapted from torch.nn.Embedding, note that we pad the vocabulary size to -- make sure it is divisible by the number of model parallel GPUs. -- -- In order to support various loading methods, we ensure that LoRA-added -- embeddings are always at the end of TP-sharded tensors. In other words, -- we shard base embeddings and LoRA embeddings separately (both padded), -- and place them in the same tensor. -- In this example, we will have the original vocab size = 1010, -- added vocab size = 16 and padding to 64. Therefore, the total -- vocab size with padding will be 1088 (because we first pad 1010 to -- 1024, add 16, and then pad to 1088). -- Therefore, the tensor format looks like the following: -- TP1, rank 0 (no sharding): -- |< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >| -- corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1025 | -1 | ... | -1 | -- index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 | -- -- TP2, rank 0: -- |< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >| -- corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1010 | ... | 1025 | -1 | ... | -1 | -- index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 | -- TP2, rank 1: -- |< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >| -- corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 | -- index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 | -- -- Args: -- num_embeddings: vocabulary size. -- embedding_dim: size of hidden state. -- params_dtype: type of the parameters. -- org_num_embeddings: original vocabulary size (without LoRA). -- padding_size: padding size for the vocabulary. -- quant_config: quant config for the layer -- prefix: full name of the layer in the state dict -- disable_tp: If true, tensor parallelism will be disabled for this layer. -- """ # noqa: E501 -- -- # --8<-- [end:vocab_parallel_embedding] -- -- def __init__( -- self, -- num_embeddings: int, -- embedding_dim: int, -- params_dtype: torch.dtype | None = None, -- org_num_embeddings: int | None = None, -- padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, -- quant_config: QuantizationConfig | None = None, -- prefix: str = "", +from typing import Literal -+ -+import torch -+import torch.nn.functional as F -+from torch.nn.parameter import Parameter -+ -+import vllm.envs as envs -+from vllm.distributed import ( -+ divide, -+ get_tensor_model_parallel_rank, -+ get_tensor_model_parallel_world_size, -+ tensor_model_parallel_all_reduce, -+) -+from vllm.model_executor.custom_op import PluggableLayer -+from vllm.model_executor.determinism.batch_invariant import ( -+ linear_batch_invariant, -+) -+from vllm.model_executor.layers.quantization.base_config import ( -+ QuantizationConfig, -+ QuantizeMethodBase, -+ method_has_implemented_embedding, -+) -+from vllm.model_executor.layers.utils import dispatch_unquantized_gemm -+from vllm.model_executor.parameter import BasevLLMParameter -+from vllm.model_executor.utils import set_weight_attrs -+from vllm.platforms import current_platform -+ -+DEFAULT_VOCAB_PADDING_SIZE = 64 -+ -+ -+class UnquantizedEmbeddingMethod(QuantizeMethodBase): -+ """Unquantized method for embeddings.""" -+ -+ def create_weights( -+ self, -+ layer: torch.nn.Module, -+ input_size_per_partition: int, -+ output_partition_sizes: list[int], -+ input_size: int, -+ output_size: int, -+ params_dtype: torch.dtype, -+ **extra_weight_attrs, -+ ): -+ """Create weights for embedding layer.""" -+ weight = Parameter( -+ torch.empty( -+ sum(output_partition_sizes), -+ input_size_per_partition, -+ dtype=params_dtype, -+ ), -+ requires_grad=False, -+ ) -+ set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) -+ layer.register_parameter("weight", weight) -+ set_weight_attrs(weight, extra_weight_attrs) -+ -+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: -+ if current_platform.is_cpu(): -+ from vllm.model_executor.layers.utils import dispatch_cpu_unquantized_gemm -+ -+ dispatch_cpu_unquantized_gemm(layer, remove_weight=False) -+ -+ def apply( -+ self, -+ layer: torch.nn.Module, -+ x: torch.Tensor, -+ bias: torch.Tensor | None = None, -+ ) -> torch.Tensor: -+ if envs.VLLM_BATCH_INVARIANT and current_platform.is_cuda_alike(): -+ return linear_batch_invariant(x, layer.weight, bias) -+ return dispatch_unquantized_gemm()(layer, x, layer.weight, bias) -+ -+ def embedding(self, layer: torch.nn.Module, input_: torch.Tensor) -> torch.Tensor: -+ return F.embedding(input_, layer.weight) -+ -+ def tie_weights( -+ self, layer: torch.nn.Module, embed_tokens: "VocabParallelEmbedding" -+ ): -+ layer.weight = embed_tokens.weight -+ return layer -+ -+ -+def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int: -+ """Pad the vocab size to the given value.""" -+ return ((vocab_size + pad_to - 1) // pad_to) * pad_to -+ -+ -+def vocab_range_from_per_partition_vocab_size( -+ per_partition_vocab_size: int, rank: int, offset: int = 0 -+) -> Sequence[int]: -+ index_f = rank * per_partition_vocab_size -+ index_l = index_f + per_partition_vocab_size -+ return index_f + offset, index_l + offset -+ -+ -+def vocab_range_from_global_vocab_size( -+ global_vocab_size: int, rank: int, world_size: int, offset: int = 0 -+) -> Sequence[int]: -+ per_partition_vocab_size = divide(global_vocab_size, world_size) -+ return vocab_range_from_per_partition_vocab_size( -+ per_partition_vocab_size, rank, offset=offset -+ ) -+ -+ -+@dataclass -+class VocabParallelEmbeddingShardIndices: -+ """Indices for a shard of a vocab parallel embedding.""" -+ -+ padded_org_vocab_start_index: int -+ padded_org_vocab_end_index: int -+ padded_added_vocab_start_index: int -+ padded_added_vocab_end_index: int -+ -+ org_vocab_start_index: int -+ org_vocab_end_index: int -+ added_vocab_start_index: int -+ added_vocab_end_index: int -+ -+ @property -+ def num_org_elements(self) -> int: -+ return self.org_vocab_end_index - self.org_vocab_start_index -+ -+ @property -+ def num_added_elements(self) -> int: -+ return self.added_vocab_end_index - self.added_vocab_start_index -+ -+ @property -+ def num_org_elements_padded(self) -> int: -+ return self.padded_org_vocab_end_index - self.padded_org_vocab_start_index -+ -+ @property -+ def num_added_elements_padded(self) -> int: -+ return self.padded_added_vocab_end_index - self.padded_added_vocab_start_index -+ -+ @property -+ def num_org_vocab_padding(self) -> int: -+ return self.num_org_elements_padded - self.num_org_elements -+ -+ @property -+ def num_added_vocab_padding(self) -> int: -+ return self.num_added_elements_padded - self.num_added_elements -+ -+ @property -+ def num_elements_padded(self) -> int: -+ return self.num_org_elements_padded + self.num_added_elements_padded -+ -+ def __post_init__(self): -+ # sanity checks -+ assert self.padded_org_vocab_start_index <= self.padded_org_vocab_end_index -+ assert self.padded_added_vocab_start_index <= self.padded_added_vocab_end_index -+ -+ assert self.org_vocab_start_index <= self.org_vocab_end_index -+ assert self.added_vocab_start_index <= self.added_vocab_end_index -+ -+ assert self.org_vocab_start_index <= self.padded_org_vocab_start_index -+ assert self.added_vocab_start_index <= self.padded_added_vocab_start_index -+ assert self.org_vocab_end_index <= self.padded_org_vocab_end_index -+ assert self.added_vocab_end_index <= self.padded_added_vocab_end_index -+ -+ assert self.num_org_elements <= self.num_org_elements_padded -+ assert self.num_added_elements <= self.num_added_elements_padded -+ -+ -+@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend) -+def get_masked_input_and_mask( -+ input_: torch.Tensor, -+ org_vocab_start_index: int, -+ org_vocab_end_index: int, -+ num_org_vocab_padding: int, -+ added_vocab_start_index: int, -+ added_vocab_end_index: int, -+) -> tuple[torch.Tensor, torch.Tensor]: -+ # torch.compile will fuse all of the pointwise ops below -+ # into a single kernel, making it very fast -+ org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ < org_vocab_end_index) -+ added_vocab_mask = (input_ >= added_vocab_start_index) & ( -+ input_ < added_vocab_end_index -+ ) -+ added_offset = ( -+ added_vocab_start_index -+ - (org_vocab_end_index - org_vocab_start_index) -+ - num_org_vocab_padding -+ ) -+ valid_offset = (org_vocab_start_index * org_vocab_mask) + ( -+ added_offset * added_vocab_mask -+ ) -+ vocab_mask = org_vocab_mask | added_vocab_mask -+ input_ = vocab_mask * (input_ - valid_offset) -+ return input_, ~vocab_mask -+ -+ -+# --8<-- [start:vocab_parallel_embedding] -+@PluggableLayer.register("vocab_parallel_embedding") -+class VocabParallelEmbedding(PluggableLayer): -+ """Embedding parallelized in the vocabulary dimension. -+ -+ Adapted from torch.nn.Embedding, note that we pad the vocabulary size to -+ make sure it is divisible by the number of model parallel GPUs. -+ -+ In order to support various loading methods, we ensure that LoRA-added -+ embeddings are always at the end of TP-sharded tensors. In other words, -+ we shard base embeddings and LoRA embeddings separately (both padded), -+ and place them in the same tensor. -+ In this example, we will have the original vocab size = 1010, -+ added vocab size = 16 and padding to 64. Therefore, the total -+ vocab size with padding will be 1088 (because we first pad 1010 to -+ 1024, add 16, and then pad to 1088). -+ Therefore, the tensor format looks like the following: -+ TP1, rank 0 (no sharding): -+ |< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >| -+ corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1025 | -1 | ... | -1 | -+ index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 | -+ -+ TP2, rank 0: -+ |< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >| -+ corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1010 | ... | 1025 | -1 | ... | -1 | -+ index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 | -+ TP2, rank 1: -+ |< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >| -+ corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 | -+ index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 528 | ... | 543 | -+ -+ Args: -+ num_embeddings: vocabulary size. -+ embedding_dim: size of hidden state. -+ params_dtype: type of the parameters. -+ org_num_embeddings: original vocabulary size (without LoRA). -+ padding_size: padding size for the vocabulary. -+ quant_config: quant config for the layer -+ prefix: full name of the layer in the state dict -+ disable_tp: If true, tensor parallelism will be disabled for this layer. -+ """ # noqa: E501 -+ -+ # --8<-- [end:vocab_parallel_embedding] -+ -+ def __init__( -+ self, -+ num_embeddings: int, -+ embedding_dim: int, -+ params_dtype: torch.dtype | None = None, -+ org_num_embeddings: int | None = None, -+ padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, -+ quant_config: QuantizationConfig | None = None, -+ prefix: str = "", + + import torch + import torch.nn.functional as F +@@ -248,6 +249,7 @@ + prefix: str = "", *, disable_tp: bool = False, -- ): -- super().__init__() -- -- # Keep the input dimensions. -- self.disable_tp = disable_tp -- if disable_tp: -- tp_rank, self.tp_size = 0, 1 -- else: -- tp_rank = get_tensor_model_parallel_rank() -- self.tp_size = get_tensor_model_parallel_world_size() -- self.tp_rank = tp_rank -- self.num_embeddings = num_embeddings -- self.padding_size = padding_size -- self.org_vocab_size = org_num_embeddings or num_embeddings -- num_added_embeddings = num_embeddings - self.org_vocab_size -- self.org_vocab_size_padded = pad_vocab_size( -- self.org_vocab_size, self.padding_size -- ) -- self.num_embeddings_padded = pad_vocab_size( -- self.org_vocab_size_padded + num_added_embeddings, self.padding_size -- ) -- assert self.org_vocab_size_padded <= self.num_embeddings_padded -- -- self.shard_indices = self._get_indices( -- self.num_embeddings_padded, -- self.org_vocab_size_padded, -- self.num_embeddings, -- self.org_vocab_size, -- tp_rank, -- self.tp_size, -- ) -- self.embedding_dim = embedding_dim -- -- quant_method = None -- if quant_config is not None: -- quant_method = quant_config.get_quant_method(self, prefix=prefix) -- if quant_method is None: -- quant_method = UnquantizedEmbeddingMethod() -- -- # If we are making an embedding layer, then our quantization linear -- # method must implement the embedding operation. If we are another + lm_head_quantization: Literal["nvfp4"] | None = None, -+ ): -+ super().__init__() -+ -+ # Keep the input dimensions. -+ self.disable_tp = disable_tp -+ if disable_tp: -+ tp_rank, self.tp_size = 0, 1 -+ else: -+ tp_rank = get_tensor_model_parallel_rank() -+ self.tp_size = get_tensor_model_parallel_world_size() -+ self.tp_rank = tp_rank -+ self.num_embeddings = num_embeddings -+ self.padding_size = padding_size -+ self.org_vocab_size = org_num_embeddings or num_embeddings -+ num_added_embeddings = num_embeddings - self.org_vocab_size -+ self.org_vocab_size_padded = pad_vocab_size( -+ self.org_vocab_size, self.padding_size -+ ) -+ self.num_embeddings_padded = pad_vocab_size( -+ self.org_vocab_size_padded + num_added_embeddings, self.padding_size -+ ) -+ assert self.org_vocab_size_padded <= self.num_embeddings_padded -+ -+ self.shard_indices = self._get_indices( -+ self.num_embeddings_padded, -+ self.org_vocab_size_padded, -+ self.num_embeddings, -+ self.org_vocab_size, -+ tp_rank, -+ self.tp_size, -+ ) -+ self.embedding_dim = embedding_dim -+ -+ quant_method = None -+ if quant_config is not None: -+ quant_method = quant_config.get_quant_method(self, prefix=prefix) -+ if quant_method is None: -+ quant_method = UnquantizedEmbeddingMethod() -+ -+ # If we are making an embedding layer, then our quantization linear -+ # method must implement the embedding operation. If we are another + ): + super().__init__() + +@@ -291,6 +293,25 @@ + # method must implement the embedding operation. If we are another # layer type like ParallelLMHead, this is not important. is_embedding_layer = not isinstance(self, ParallelLMHead) -- quant_method_implements_embedding = method_has_implemented_embedding( -- type(quant_method) -- ) -- if is_embedding_layer and not quant_method_implements_embedding: -- raise NotImplementedError( -- f"The class {type(quant_method).__name__} must implement " -- "the 'embedding' method, see UnquantizedEmbeddingMethod." + self.runtime_lm_head_quantization: Literal["nvfp4"] | None = None + if not is_embedding_layer and lm_head_quantization is not None: + from vllm.model_executor.layers.linear import UnquantizedLinearMethod + from vllm.model_executor.layers.quantization.online.nvfp4 import ( + Nvfp4OnlineLinearMethod, - ) - -- self.quant_method: QuantizeMethodBase = quant_method -- -- if params_dtype is None: -- params_dtype = torch.get_default_dtype() -- self.params_dtype = params_dtype -- # Divide the weight matrix along the vocabulary dimension. -- self.num_added_embeddings = self.num_embeddings - self.org_vocab_size -- self.num_embeddings_per_partition = divide( -- self.num_embeddings_padded, self.tp_size -- ) -- assert ( -- self.shard_indices.num_elements_padded == self.num_embeddings_per_partition -- ) -- self.num_org_embeddings_per_partition = ( -- self.shard_indices.org_vocab_end_index -- - self.shard_indices.org_vocab_start_index -- ) -- self.num_added_embeddings_per_partition = ( -- self.shard_indices.added_vocab_end_index -- - self.shard_indices.added_vocab_start_index -- ) -- -- self.quant_method.create_weights( -- self, -- self.embedding_dim, -- [self.num_embeddings_per_partition], -- self.embedding_dim, -- self.num_embeddings_padded, -- params_dtype=params_dtype, -- weight_loader=self.weight_loader, -- ) -- self.update_param_tp_status() -- -- def update_param_tp_status(self): -- for param in self.parameters(): -- if isinstance(param, BasevLLMParameter): -- param.tp_rank = self.tp_rank -- param.tp_size = self.tp_size -- -- @classmethod -- def _get_indices( -- cls, -- vocab_size_padded: int, -- org_vocab_size_padded: int, -- vocab_size: int, -- org_vocab_size: int, -- tp_rank: int, -- tp_size: int, -- ) -> VocabParallelEmbeddingShardIndices: -- """Get start and end indices for vocab parallel embedding, following the -- layout outlined in the class docstring, based on the given tp_rank and -- tp_size.""" -- num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded -- padded_org_vocab_start_index, padded_org_vocab_end_index = ( -- vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, tp_size) -- ) -- padded_added_vocab_start_index, padded_added_vocab_end_index = ( -- vocab_range_from_global_vocab_size( -- num_added_embeddings_padded, tp_rank, tp_size, offset=org_vocab_size -- ) -- ) -- # remove padding -- org_vocab_start_index = min(padded_org_vocab_start_index, org_vocab_size) -- org_vocab_end_index = min(padded_org_vocab_end_index, org_vocab_size) -- added_vocab_start_index = min(padded_added_vocab_start_index, vocab_size) -- added_vocab_end_index = min(padded_added_vocab_end_index, vocab_size) -- return VocabParallelEmbeddingShardIndices( -- padded_org_vocab_start_index, -- padded_org_vocab_end_index, -- padded_added_vocab_start_index, -- padded_added_vocab_end_index, -- org_vocab_start_index, -- org_vocab_end_index, -- added_vocab_start_index, -- added_vocab_end_index, -- ) -- -- def get_sharded_to_full_mapping(self) -> list[int] | None: -- """Get a mapping that can be used to reindex the gathered -- logits for sampling. -- -- During sampling, we gather logits from all ranks. The relationship -- of index->token_id will follow the same format as outlined in the class -- docstring. However, after the gather, we want to reindex the final -- logits tensor to map index->token_id one-to-one (the index is always -- equal the token_id it corresponds to). The indices returned by this -- method allow us to do that. -- """ -- if self.tp_size < 2: -- return None -- -- base_embeddings: list[int] = [] -- added_embeddings: list[int] = [] -- padding: list[int] = [] -- for tp_rank in range(self.tp_size): -- shard_indices = self._get_indices( -- self.num_embeddings_padded, -- self.org_vocab_size_padded, -- self.num_embeddings, -- self.org_vocab_size, -- tp_rank, -- self.tp_size, -- ) -- range_start = self.num_embeddings_per_partition * tp_rank -- range_end = self.num_embeddings_per_partition * (tp_rank + 1) -- base_embeddings.extend( -- range(range_start, range_start + shard_indices.num_org_elements) -- ) -- padding.extend( -- range( -- range_start + shard_indices.num_org_elements, -- range_start + shard_indices.num_org_elements_padded, -- ) -- ) -- added_embeddings.extend( -- range( -- range_start + shard_indices.num_org_elements_padded, -- range_start -- + shard_indices.num_org_elements_padded -- + shard_indices.num_added_elements, -- ) -- ) -- padding.extend( -- range( -- range_start -- + shard_indices.num_org_elements_padded -- + shard_indices.num_added_elements, -- range_start -- + shard_indices.num_org_elements_padded -- + shard_indices.num_added_elements_padded, -- ) -- ) -- assert ( -- range_start -- + shard_indices.num_org_elements_padded -- + shard_indices.num_added_elements_padded -- == range_end -- ) -- ret = base_embeddings + added_embeddings + padding -- assert len(ret) == self.num_embeddings_padded -- return ret -- -- def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): -- output_dim = getattr(param, "output_dim", None) -- packed_dim = getattr(param, "packed_dim", None) -- -- # If parameter does not have output dim, then it should -- # be copied onto all gpus (e.g. g_idx for act_order gptq). -- if output_dim is None: -- if ( -- loaded_weight.ndim == 0 -- and param.data.ndim == 1 -- and param.data.numel() == 1 ++ ) ++ + if not isinstance( + quant_method, + (UnquantizedEmbeddingMethod, UnquantizedLinearMethod), - ): -- loaded_weight = loaded_weight.reshape(1) -- assert param.data.shape == loaded_weight.shape -- param.data.copy_(loaded_weight) -- return -- -- # Shard indexes for loading the weight -- start_idx = self.shard_indices.org_vocab_start_index -- shard_size = self.shard_indices.org_vocab_end_index - start_idx -- -- # If param packed on the same dim we are sharding on, then -- # need to adjust offsets of loaded weight by pack_factor. -- if packed_dim is not None and packed_dim == output_dim: -- packed_factor = ( -- param.packed_factor -- if isinstance(param, BasevLLMParameter) -- else param.pack_factor -- ) -- assert loaded_weight.shape[output_dim] == ( -- self.org_vocab_size // param.packed_factor -- ) -- start_idx = start_idx // packed_factor -- shard_size = shard_size // packed_factor -- else: -- assert loaded_weight.shape[output_dim] == self.org_vocab_size -- -- # Copy the data. Select chunk corresponding to current shard. -- loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) -- param[: loaded_weight.shape[0]].data.copy_(loaded_weight) -- param[loaded_weight.shape[0] :].data.fill_(0) -- -- def forward(self, input_): -- if self.tp_size > 1: -- # Build the mask. -- masked_input, input_mask = get_masked_input_and_mask( -- input_, -- self.shard_indices.org_vocab_start_index, -- self.shard_indices.org_vocab_end_index, -- self.shard_indices.num_org_vocab_padding, -- self.shard_indices.added_vocab_start_index, -- self.shard_indices.added_vocab_end_index, -+ raise ValueError( -+ "Runtime NVFP4 proposal-head quantization requires an " -+ "unquantized checkpoint head" -+ ) -+ quant_method = Nvfp4OnlineLinearMethod( -+ use_a16=envs.VLLM_LM_HEAD_A16 - ) -- else: -- masked_input = input_ -- # Get the embeddings. -- output_parallel = self.quant_method.embedding(self, masked_input.long()) -- # Mask the output embedding. -- if self.tp_size > 1: -- output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0) -- # Reduce across all the model parallel GPUs. -- return tensor_model_parallel_all_reduce(output_parallel) -- return output_parallel -- -- def extra_repr(self) -> str: -- s = f"num_embeddings={self.num_embeddings}" -- s += f", num_embeddings_per_partition={self.num_embeddings_per_partition}" -- s += f", embedding_dim={self.embedding_dim}" -- s += f", org_vocab_size={self.org_vocab_size}" -- s += f", num_embeddings_padded={self.num_embeddings_padded}" -- s += f", tp_size={self.tp_size}" -- return s -- -- --# --8<-- [start:parallel_lm_head] --@PluggableLayer.register("parallel_lm_head") --class ParallelLMHead(VocabParallelEmbedding): -- """Parallelized LM head. -- -- Output logits weight matrices used in the Sampler. The weight and bias -- tensors are padded to make sure they are divisible by the number of -- model parallel GPUs. -- -- Args: -- num_embeddings: vocabulary size. -- embedding_dim: size of hidden state. -- bias: whether to use bias. -- params_dtype: type of the parameters. -- org_num_embeddings: original vocabulary size (without LoRA). -- padding_size: padding size for the vocabulary. -- disable_tp: If true, tensor parallelism will be disabled for this layer. -- """ -- -- # --8<-- [end:parallel_lm_head] -- -- def __init__( -- self, -- num_embeddings: int, -- embedding_dim: int, -- bias: bool = False, -- params_dtype: torch.dtype | None = None, -- org_num_embeddings: int | None = None, -- padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, -- quant_config: QuantizationConfig | None = None, -- prefix: str = "", -+ self.runtime_lm_head_quantization = "nvfp4" -+ quant_method_implements_embedding = method_has_implemented_embedding( -+ type(quant_method) -+ ) -+ if is_embedding_layer and not quant_method_implements_embedding: -+ raise NotImplementedError( -+ f"The class {type(quant_method).__name__} must implement " -+ "the 'embedding' method, see UnquantizedEmbeddingMethod." -+ ) -+ -+ self.quant_method: QuantizeMethodBase = quant_method -+ -+ if params_dtype is None: -+ params_dtype = torch.get_default_dtype() -+ self.params_dtype = params_dtype -+ # Divide the weight matrix along the vocabulary dimension. -+ self.num_added_embeddings = self.num_embeddings - self.org_vocab_size -+ self.num_embeddings_per_partition = divide( -+ self.num_embeddings_padded, self.tp_size -+ ) -+ assert ( -+ self.shard_indices.num_elements_padded == self.num_embeddings_per_partition -+ ) -+ self.num_org_embeddings_per_partition = ( -+ self.shard_indices.org_vocab_end_index -+ - self.shard_indices.org_vocab_start_index -+ ) -+ self.num_added_embeddings_per_partition = ( -+ self.shard_indices.added_vocab_end_index -+ - self.shard_indices.added_vocab_start_index -+ ) -+ -+ self.quant_method.create_weights( -+ self, -+ self.embedding_dim, -+ [self.num_embeddings_per_partition], -+ self.embedding_dim, -+ self.num_embeddings_padded, -+ params_dtype=params_dtype, -+ weight_loader=self.weight_loader, -+ ) -+ self.update_param_tp_status() -+ -+ def update_param_tp_status(self): -+ for param in self.parameters(): -+ if isinstance(param, BasevLLMParameter): -+ param.tp_rank = self.tp_rank -+ param.tp_size = self.tp_size -+ -+ @classmethod -+ def _get_indices( -+ cls, -+ vocab_size_padded: int, -+ org_vocab_size_padded: int, -+ vocab_size: int, -+ org_vocab_size: int, -+ tp_rank: int, -+ tp_size: int, -+ ) -> VocabParallelEmbeddingShardIndices: -+ """Get start and end indices for vocab parallel embedding, following the -+ layout outlined in the class docstring, based on the given tp_rank and -+ tp_size.""" -+ num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded -+ padded_org_vocab_start_index, padded_org_vocab_end_index = ( -+ vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, tp_size) -+ ) -+ padded_added_vocab_start_index, padded_added_vocab_end_index = ( -+ vocab_range_from_global_vocab_size( -+ num_added_embeddings_padded, tp_rank, tp_size, offset=org_vocab_size -+ ) -+ ) -+ # remove padding -+ org_vocab_start_index = min(padded_org_vocab_start_index, org_vocab_size) -+ org_vocab_end_index = min(padded_org_vocab_end_index, org_vocab_size) -+ added_vocab_start_index = min(padded_added_vocab_start_index, vocab_size) -+ added_vocab_end_index = min(padded_added_vocab_end_index, vocab_size) -+ return VocabParallelEmbeddingShardIndices( -+ padded_org_vocab_start_index, -+ padded_org_vocab_end_index, -+ padded_added_vocab_start_index, -+ padded_added_vocab_end_index, -+ org_vocab_start_index, -+ org_vocab_end_index, -+ added_vocab_start_index, -+ added_vocab_end_index, -+ ) -+ -+ def get_sharded_to_full_mapping(self) -> list[int] | None: -+ """Get a mapping that can be used to reindex the gathered -+ logits for sampling. -+ -+ During sampling, we gather logits from all ranks. The relationship -+ of index->token_id will follow the same format as outlined in the class -+ docstring. However, after the gather, we want to reindex the final -+ logits tensor to map index->token_id one-to-one (the index is always -+ equal the token_id it corresponds to). The indices returned by this -+ method allow us to do that. -+ """ -+ if self.tp_size < 2: -+ return None -+ -+ base_embeddings: list[int] = [] -+ added_embeddings: list[int] = [] -+ padding: list[int] = [] -+ for tp_rank in range(self.tp_size): -+ shard_indices = self._get_indices( -+ self.num_embeddings_padded, -+ self.org_vocab_size_padded, -+ self.num_embeddings, -+ self.org_vocab_size, -+ tp_rank, -+ self.tp_size, -+ ) -+ range_start = self.num_embeddings_per_partition * tp_rank -+ range_end = self.num_embeddings_per_partition * (tp_rank + 1) -+ base_embeddings.extend( -+ range(range_start, range_start + shard_indices.num_org_elements) -+ ) -+ padding.extend( -+ range( -+ range_start + shard_indices.num_org_elements, -+ range_start + shard_indices.num_org_elements_padded, -+ ) -+ ) -+ added_embeddings.extend( -+ range( -+ range_start + shard_indices.num_org_elements_padded, -+ range_start -+ + shard_indices.num_org_elements_padded -+ + shard_indices.num_added_elements, -+ ) -+ ) -+ padding.extend( -+ range( -+ range_start -+ + shard_indices.num_org_elements_padded -+ + shard_indices.num_added_elements, -+ range_start -+ + shard_indices.num_org_elements_padded -+ + shard_indices.num_added_elements_padded, -+ ) -+ ) -+ assert ( -+ range_start -+ + shard_indices.num_org_elements_padded -+ + shard_indices.num_added_elements_padded -+ == range_end -+ ) -+ ret = base_embeddings + added_embeddings + padding -+ assert len(ret) == self.num_embeddings_padded -+ return ret -+ -+ def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): -+ output_dim = getattr(param, "output_dim", None) -+ packed_dim = getattr(param, "packed_dim", None) -+ -+ # If parameter does not have output dim, then it should -+ # be copied onto all gpus (e.g. g_idx for act_order gptq). -+ if output_dim is None: -+ if ( -+ loaded_weight.ndim == 0 -+ and param.data.ndim == 1 -+ and param.data.numel() == 1 -+ ): -+ loaded_weight = loaded_weight.reshape(1) -+ assert param.data.shape == loaded_weight.shape -+ param.data.copy_(loaded_weight) -+ return -+ -+ # Shard indexes for loading the weight -+ start_idx = self.shard_indices.org_vocab_start_index -+ shard_size = self.shard_indices.org_vocab_end_index - start_idx -+ -+ # If param packed on the same dim we are sharding on, then -+ # need to adjust offsets of loaded weight by pack_factor. -+ if packed_dim is not None and packed_dim == output_dim: -+ packed_factor = ( -+ param.packed_factor -+ if isinstance(param, BasevLLMParameter) -+ else param.pack_factor -+ ) -+ assert loaded_weight.shape[output_dim] == ( -+ self.org_vocab_size // param.packed_factor -+ ) -+ start_idx = start_idx // packed_factor -+ shard_size = shard_size // packed_factor -+ else: -+ assert loaded_weight.shape[output_dim] == self.org_vocab_size -+ -+ # Copy the data. Select chunk corresponding to current shard. -+ loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) -+ param[: loaded_weight.shape[0]].data.copy_(loaded_weight) -+ param[loaded_weight.shape[0] :].data.fill_(0) -+ -+ def forward(self, input_): -+ if self.tp_size > 1: -+ # Build the mask. -+ masked_input, input_mask = get_masked_input_and_mask( -+ input_, -+ self.shard_indices.org_vocab_start_index, -+ self.shard_indices.org_vocab_end_index, -+ self.shard_indices.num_org_vocab_padding, -+ self.shard_indices.added_vocab_start_index, -+ self.shard_indices.added_vocab_end_index, -+ ) -+ else: -+ masked_input = input_ -+ # Get the embeddings. -+ output_parallel = self.quant_method.embedding(self, masked_input.long()) -+ # Mask the output embedding. -+ if self.tp_size > 1: -+ output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0) -+ # Reduce across all the model parallel GPUs. -+ return tensor_model_parallel_all_reduce(output_parallel) -+ return output_parallel -+ -+ def extra_repr(self) -> str: -+ s = f"num_embeddings={self.num_embeddings}" -+ s += f", num_embeddings_per_partition={self.num_embeddings_per_partition}" -+ s += f", embedding_dim={self.embedding_dim}" -+ s += f", org_vocab_size={self.org_vocab_size}" -+ s += f", num_embeddings_padded={self.num_embeddings_padded}" -+ s += f", tp_size={self.tp_size}" -+ return s -+ -+ -+# --8<-- [start:parallel_lm_head] -+@PluggableLayer.register("parallel_lm_head") -+class ParallelLMHead(VocabParallelEmbedding): -+ """Parallelized LM head. -+ -+ Output logits weight matrices used in the Sampler. The weight and bias -+ tensors are padded to make sure they are divisible by the number of -+ model parallel GPUs. -+ -+ Args: -+ num_embeddings: vocabulary size. -+ embedding_dim: size of hidden state. -+ bias: whether to use bias. -+ params_dtype: type of the parameters. -+ org_num_embeddings: original vocabulary size (without LoRA). -+ padding_size: padding size for the vocabulary. -+ disable_tp: If true, tensor parallelism will be disabled for this layer. -+ """ -+ -+ # --8<-- [end:parallel_lm_head] -+ -+ def __init__( -+ self, -+ num_embeddings: int, -+ embedding_dim: int, -+ bias: bool = False, -+ params_dtype: torch.dtype | None = None, -+ org_num_embeddings: int | None = None, -+ padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, -+ quant_config: QuantizationConfig | None = None, -+ prefix: str = "", - *, - disable_tp: bool = False, -- ): -- super().__init__( -- num_embeddings, -- embedding_dim, -- params_dtype, -- org_num_embeddings, -- padding_size, -- quant_config, -+ lm_head_quantization: Literal["nvfp4"] | None = None, -+ ): -+ super().__init__( -+ num_embeddings, -+ embedding_dim, -+ params_dtype, -+ org_num_embeddings, -+ padding_size, -+ quant_config, - prefix, - disable_tp=disable_tp, -+ lm_head_quantization=lm_head_quantization, - ) -- self.quant_config = quant_config -- if bias: -- self._register_bias() -- else: -- self.register_parameter("bias", None) -- -- def _register_bias(self): -- data = torch.empty(self.num_embeddings_per_partition, dtype=self.params_dtype) -- self.bias = Parameter(data, requires_grad=False) -- weight_attrs = dict(output_dim=0, weight_loader=self.weight_loader) -- set_weight_attrs(weight=self.bias, weight_attrs=weight_attrs) -- -- def tie_weights(self, embed_tokens: VocabParallelEmbedding): -- """Tie the weights with word embeddings.""" -- return self.quant_method.tie_weights(self, embed_tokens) -- -- def forward(self, input_): -- del input_ -- raise RuntimeError("LMHead's weights should be used in the sampler.") -+ self.quant_config = quant_config -+ if bias: -+ self._register_bias() -+ else: -+ self.register_parameter("bias", None) -+ -+ def _register_bias(self): -+ data = torch.empty(self.num_embeddings_per_partition, dtype=self.params_dtype) -+ self.bias = Parameter(data, requires_grad=False) -+ weight_attrs = dict(output_dim=0, weight_loader=self.weight_loader) -+ set_weight_attrs(weight=self.bias, weight_attrs=weight_attrs) -+ -+ def tie_weights(self, embed_tokens: VocabParallelEmbedding): -+ """Tie the weights with word embeddings.""" -+ return self.quant_method.tie_weights(self, embed_tokens) -+ -+ def forward(self, input_): -+ del input_ -+ raise RuntimeError("LMHead's weights should be used in the sampler.") -diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py -index 8ac43e0bb2a17957ac3f013c4da75d030fae0502..e200fd64483419ee72adc107be580519fbfb38f8 100644 ---- a/vllm/model_executor/models/deepseek_mtp.py -+++ b/vllm/model_executor/models/deepseek_mtp.py -@@ -1,547 +1,550 @@ --# SPDX-License-Identifier: Apache-2.0 --# SPDX-FileCopyrightText: Copyright contributors to the vLLM project --import typing --from collections.abc import Callable, Iterable -- --import torch --import torch.nn as nn --from transformers import PretrainedConfig -- --from vllm.compilation.decorators import support_torch_compile --from vllm.config import VllmConfig --from vllm.distributed import tensor_model_parallel_all_gather --from vllm.model_executor.layers.fused_moe import ( -- fused_moe_make_expert_params_mapping, --) --from vllm.model_executor.layers.fused_moe.utils import ( -- is_model_fused_shared_expert_compatible, --) --from vllm.model_executor.layers.layernorm import RMSNorm --from vllm.model_executor.layers.logits_processor import LogitsProcessor --from vllm.model_executor.layers.quantization import QuantizationConfig --from vllm.model_executor.layers.vocab_parallel_embedding import ( -- ParallelLMHead, -- VocabParallelEmbedding, --) --from vllm.model_executor.model_loader.mtp_validation import ( -- is_mtp_completeness_check_enabled, --) --from vllm.model_executor.model_loader.weight_utils import ( -- default_weight_loader, -- maybe_remap_kv_scale_name, --) --from vllm.platforms import current_platform --from vllm.sequence import IntermediateTensors -- --from .deepseek_v2 import ( -- DeepseekV2DecoderLayer, -- DeepseekV2MixtureOfExperts, -- DeepseekV2MoE, -- _try_load_fp8_indexer_wk, --) --from .utils import ( -- get_pp_missing_layer_names, -- get_spec_layer_idx_from_weight_name, -- maybe_prefix, --) -- -- --class SharedHead(nn.Module): -+# SPDX-License-Identifier: Apache-2.0 -+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -+import typing -+from collections.abc import Callable, Iterable -+ -+import torch -+import torch.nn as nn -+from transformers import PretrainedConfig -+ -+from vllm.compilation.decorators import support_torch_compile -+from vllm.config import VllmConfig -+from vllm.distributed import tensor_model_parallel_all_gather -+from vllm.model_executor.layers.fused_moe import ( -+ fused_moe_make_expert_params_mapping, -+) -+from vllm.model_executor.layers.fused_moe.utils import ( -+ is_model_fused_shared_expert_compatible, -+) -+from vllm.model_executor.layers.layernorm import RMSNorm -+from vllm.model_executor.layers.logits_processor import LogitsProcessor -+from vllm.model_executor.layers.quantization import QuantizationConfig -+from vllm.model_executor.layers.vocab_parallel_embedding import ( -+ ParallelLMHead, -+ VocabParallelEmbedding, -+) -+from vllm.model_executor.model_loader.mtp_validation import ( -+ is_mtp_completeness_check_enabled, -+) -+from vllm.model_executor.model_loader.weight_utils import ( -+ default_weight_loader, -+ maybe_remap_kv_scale_name, -+) -+from vllm.platforms import current_platform -+from vllm.sequence import IntermediateTensors -+ -+from .deepseek_v2 import ( -+ DeepseekV2DecoderLayer, -+ DeepseekV2MixtureOfExperts, -+ DeepseekV2MoE, -+ _try_load_fp8_indexer_wk, -+) -+from .utils import ( -+ get_pp_missing_layer_names, -+ get_spec_layer_idx_from_weight_name, -+ maybe_prefix, -+) -+ -+ -+class SharedHead(nn.Module): - def __init__( - self, - config: PretrainedConfig, - prefix: str, - quant_config: QuantizationConfig | None = None, -+ *, -+ lm_head_quantization: typing.Literal["nvfp4"] | None = None, - ) -> None: -- super().__init__() -- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) -- self.head = ParallelLMHead( -- config.vocab_size, -+ super().__init__() -+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) -+ self.head = ParallelLMHead( -+ config.vocab_size, - config.hidden_size, - quant_config=quant_config, - prefix=maybe_prefix(prefix, "head"), -+ lm_head_quantization=lm_head_quantization, - ) -- -- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: -- return self.norm(hidden_states) -- -- --class DeepSeekMultiTokenPredictorLayer(nn.Module): -- def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: -- super().__init__() -- -- assert vllm_config.speculative_config is not None -- config = vllm_config.speculative_config.draft_model_config.hf_config -- self.config = config -- quant_config = vllm_config.quant_config -- -- self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) -- self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) -- self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) -- -- self.device = current_platform.device_type -- -- self.is_v32 = hasattr(config, "index_topk") -- if self.is_v32: -- topk_tokens = config.index_topk -- topk_indices_buffer = torch.empty( -- vllm_config.scheduler_config.max_num_batched_tokens, -- topk_tokens, -- dtype=torch.int32, -- device=self.device, -- ) -- else: -- topk_indices_buffer = None -- -- self.shared_head = SharedHead( -- config=config, prefix=prefix, quant_config=quant_config -- ) -- self.mtp_block = DeepseekV2DecoderLayer( -- vllm_config, -- prefix, -- config=self.config, -- topk_indices_buffer=topk_indices_buffer, -- ) -- -- def forward( -- self, -- input_ids: torch.Tensor, -- positions: torch.Tensor, -- previous_hidden_states: torch.Tensor, -- inputs_embeds: torch.Tensor | None = None, -- spec_step_index: int = 0, -- ) -> torch.Tensor: -- assert inputs_embeds is not None -- # masking inputs at position 0, as not needed by MTP -- inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds) -- inputs_embeds = self.enorm(inputs_embeds) -- previous_hidden_states = self.hnorm(previous_hidden_states) -- -- hidden_states = self.eh_proj( -- torch.cat([inputs_embeds, previous_hidden_states], dim=-1) -- ) -- -- hidden_states, residual = self.mtp_block( -- positions=positions, -- hidden_states=hidden_states, -- residual=None, -- ) -- hidden_states = residual + hidden_states # pre-final-norm (logits hidden) -- if self.mtp_block.use_sequence_parallel_moe: -- hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) -- hidden_states = hidden_states[: positions.shape[0]] -- # Recycle the post-final-norm hidden into the next draft step. -- # compute_logits applies shared_head (== final norm) to the pre-norm -- # element, so logits and the recycle each get exactly one final-norm. -- # Matches SGLang's deepseek_nextn. -- return hidden_states, self.shared_head(hidden_states) -- -- --class DeepSeekMultiTokenPredictor(nn.Module): -- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): -- super().__init__() -- config = vllm_config.model_config.hf_config -- self.mtp_start_layer_idx = config.num_hidden_layers -- self.num_mtp_layers = config.num_nextn_predict_layers -- # to map the exact layer index from weights -- -- self.layers = torch.nn.ModuleDict( -- { -- str(idx): DeepSeekMultiTokenPredictorLayer( -- vllm_config, f"{prefix}.layers.{idx}" -- ) -- for idx in range( -- self.mtp_start_layer_idx, -- self.mtp_start_layer_idx + self.num_mtp_layers, -- ) -- } -- ) -- self.embed_tokens = VocabParallelEmbedding( -- config.vocab_size, -- config.hidden_size, -- prefix=maybe_prefix(prefix, "embed_tokens"), -- ) -- self.logits_processor = LogitsProcessor(config.vocab_size) -- -- def set_skip_topk(self, skip: bool): -- """Toggle skip_topk on all MTP layers with sparse attention. -- -- Called by the proposer to implement index_share_for_mtp_iteration: -- step 0 sets skip=False (compute own indices), steps 1+ set skip=True -- (reuse step 0's indices). -- """ -- for layer in self.layers.values(): -- mtp_block = getattr(layer, "mtp_block", None) -- if mtp_block is not None: -- self_attn = getattr(mtp_block, "self_attn", None) -- if self_attn is not None: -- mla_attn = getattr(self_attn, "mla_attn", None) -- if mla_attn is not None and hasattr(mla_attn, "skip_topk"): -- mla_attn.skip_topk = skip -- -- def compact_topk_indices(self, slot_ids: torch.Tensor): -- """Gather the top-k index rows at ``slot_ids`` to the front of the buffer.""" -- num_slots = slot_ids.numel() -- for layer in self.layers.values(): -- mtp_block = getattr(layer, "mtp_block", None) -- if mtp_block is not None: -- self_attn = getattr(mtp_block, "self_attn", None) -- if self_attn is not None: -- mla_attn = getattr(self_attn, "mla_attn", None) -- if mla_attn is not None and hasattr( -- mla_attn, "topk_indices_buffer" -- ): -- topk_indices_buffer = mla_attn.topk_indices_buffer -- topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] -- -- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: -- return self.embed_tokens(input_ids) -- -- def forward( -- self, -- input_ids: torch.Tensor, -- positions: torch.Tensor, -- previous_hidden_states: torch.Tensor, -- inputs_embeds: torch.Tensor | None = None, -- spec_step_idx: int = 0, -- ) -> torch.Tensor: -- if inputs_embeds is None: -- inputs_embeds = self.embed_tokens(input_ids) -- current_step_idx = spec_step_idx % self.num_mtp_layers -- return self.layers[str(self.mtp_start_layer_idx + current_step_idx)]( -- input_ids, -- positions, -- previous_hidden_states, -- inputs_embeds, -- current_step_idx, -- ) -- -- def compute_logits( -- self, -- hidden_states: torch.Tensor, -- spec_step_idx: int = 0, -- ) -> torch.Tensor: -- current_step_idx = spec_step_idx % self.num_mtp_layers -- mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] -- logits = self.logits_processor( -- mtp_layer.shared_head.head, mtp_layer.shared_head(hidden_states) -- ) -- return logits -- -- --@support_torch_compile --class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): -- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): -- super().__init__() -- self.config = vllm_config.model_config.hf_config -- self.quant_config = vllm_config.quant_config -- self.model = DeepSeekMultiTokenPredictor( -- vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") -- ) -- # Set MoE hyperparameters -- self.set_moe_parameters() -- -- def set_moe_parameters(self): -- self.num_moe_layers = self.config.num_nextn_predict_layers -- self.num_expert_groups = self.config.n_group -- -- self.moe_layers = [] -- self.moe_mlp_layers = [] -- example_moe = None -- for layer in self.model.layers.values(): -- assert isinstance(layer, DeepSeekMultiTokenPredictorLayer) -- layer = layer.mtp_block -- assert isinstance(layer, DeepseekV2DecoderLayer) -- if isinstance(layer.mlp, DeepseekV2MoE): -- example_moe = layer.mlp -- self.moe_mlp_layers.append(layer.mlp) -- self.moe_layers.append(layer.mlp.experts) -- self.extract_moe_parameters(example_moe) -- self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( -- self.model.layers.values(), -- DeepseekV2MoE, -- "mtp_block.mlp", -- ) -- -- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: -- return self.model.embed_input_ids(input_ids) -- -- def forward( -- self, -- input_ids: torch.Tensor | None, -- positions: torch.Tensor, -- hidden_states: torch.Tensor, -- intermediate_tensors: IntermediateTensors | None = None, -- inputs_embeds: torch.Tensor | None = None, -- spec_step_idx: int = 0, -- ) -> torch.Tensor: -- hidden_states = self.model( -- input_ids, -- positions, -- hidden_states, -- inputs_embeds, -- spec_step_idx, -- ) -- return hidden_states -- -- def compute_logits( -- self, -- hidden_states: torch.Tensor, -- spec_step_idx: int = 0, -- ) -> torch.Tensor | None: -- return self.model.compute_logits(hidden_states, spec_step_idx) -- -- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: -- stacked_params_mapping = [ -- ("gate_up_proj", "gate_proj", 0), -- ("gate_up_proj", "up_proj", 1), -- ("fused_qkv_a_proj", "q_a_proj", 0), -- ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), -- ] -- -- # Fused indexer wk + weights_proj (shard 0 = wk, shard 1 = weights_proj) -- indexer_fused_mapping = [ -- ("wk_weights_proj", "wk", 0), -- ("wk_weights_proj", "weights_proj", 1), -- ] -- stacked_params_mapping.extend(indexer_fused_mapping) -- -- expert_params_mapping = fused_moe_make_expert_params_mapping( -- self, -- ckpt_gate_proj_name="gate_proj", -- ckpt_down_proj_name="down_proj", -- ckpt_up_proj_name="up_proj", -- num_experts=self.config.n_routed_experts -- + ( -- self.config.n_shared_experts -- if self.is_fused_shared_expert_enabled -- else 0 -- ), -- ) -- -- pp_missing_layer_names = get_pp_missing_layer_names(self) -- params_dict = dict(self.named_parameters()) -- loaded_params: set[str] = set() -- _pending_wk_fp8: dict = {} # FP8 indexer wk dequant buffer -- for name, loaded_weight in weights: -- if "rotary_emb.inv_freq" in name: -- continue -- spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) -- if spec_layer is None: -- continue -- is_fusion_moe_shared_experts_layer = ( -- self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) -- ) -- name = self._rewrite_spec_layer_name(spec_layer, name) -- -- if _try_load_fp8_indexer_wk( -- name, -- loaded_weight, -- _pending_wk_fp8, -- params_dict, -- loaded_params, -- pp_missing_layer_names, -- ): -- continue -- -- for param_name, weight_name, shard_id in stacked_params_mapping: -- # Skip non-stacked layers and experts (experts handled below). -- if weight_name not in name: -- continue -- # We have mlp.experts[0].gate_proj in the checkpoint. -- # Since we handle the experts below in expert_params_mapping, -- # we need to skip here BEFORE we update the name, otherwise -- # name will be updated to mlp.experts[0].gate_up_proj, which -- # will then be updated below in expert_params_mapping -- # for mlp.experts[0].gate_gate_up_proj, which breaks load. -- if ("mlp.experts." in name) and name not in params_dict: -- continue -- if is_fusion_moe_shared_experts_layer: -- continue -- name_mapped = name.replace(weight_name, param_name) -- -- # QKV fusion is optional, fall back to normal -- # weight loading if it's not enabled -- if ( -- param_name == "fused_qkv_a_proj" -- ) and name_mapped not in params_dict: -- continue -- else: -- name = name_mapped -- -- # Skip loading extra bias for GPTQ models. -- if name.endswith(".bias") and name not in params_dict: -- continue -- -- param = params_dict[name] -- weight_loader = param.weight_loader -- weight_loader(param, loaded_weight, shard_id) -- break -- else: -- # Special handling: when AITER fusion_shared_experts is enabled, -- # checkpoints may provide a single widened shared_experts tensor -- # without explicit expert indices -- # (e.g. ...mlp.shared_experts.gate_proj.weight). -- # For models with multiple shared experts, split that tensor -- # evenly into per-shared-expert slices and load them into -- # appended expert slots mlp.experts.{n_routed_experts + j}.* -- # accordingly. -- num_chunks = 1 -- if is_fusion_moe_shared_experts_layer: -- num_chunks = getattr(self.config, "n_shared_experts", 1) or 1 -- # Determine split axis based on op type -- # gate/up: ColumnParallel → split along dim 0 -- # down: RowParallel → split along dim 1 -- split_dim = ( -- 1 -- if ("down_proj.weight" in name and loaded_weight.ndim > 1) -- else 0 -- ) -- total = loaded_weight.shape[split_dim] -- assert total % num_chunks == 0, ( -- f"Shared expert weight dim {total} " -- f"not divisible by num_chunks {num_chunks}" -- ) -- chunk_size = total // num_chunks -- -- for j in range(num_chunks): -- chunk_name = name -- weight_to_load = loaded_weight -- -- if is_fusion_moe_shared_experts_layer: -- chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size) -- if loaded_weight.ndim == 1: -- weight_to_load = loaded_weight[chunk_slice] -- elif split_dim == 0: -- weight_to_load = loaded_weight[chunk_slice, :] -- else: -- weight_to_load = loaded_weight[:, chunk_slice] -- # Synthesize an expert-style name so expert mapping -- # can route it -- chunk_name = name.replace( -- "mlp.shared_experts", -- f"mlp.experts.{self.config.n_routed_experts + j}", -- ) -- -- # Use expert_params_mapping to locate the destination -- # param and delegate to its expert-aware weight_loader -- # with expert_id. -- is_expert_weight = False -- for mapping in expert_params_mapping: -- param_name, weight_name, expert_id, expert_shard_id = mapping -- if weight_name not in chunk_name: -- continue -- -- # Anyway, this is an expert weight and should not be -- # attempted to load as other weights later -- is_expert_weight = True -- -- # Do not modify `name` since the loop may continue here -- # Instead, create a new variable -- name_mapped = chunk_name.replace(weight_name, param_name) -- -- param = params_dict[name_mapped] -- # We should ask the weight loader to return success or -- # not here since otherwise we may skip experts with -- # other available replicas. -- weight_loader = typing.cast( -- Callable[..., bool], param.weight_loader -- ) -- success = weight_loader( -- param, -- weight_to_load, -- name_mapped, -- shard_id=expert_shard_id, -- expert_id=expert_id, -- return_success=True, -- ) -- if success: -- if not is_fusion_moe_shared_experts_layer: -- name = name_mapped -- else: -- loaded_params.add(name_mapped) -- break -- else: -- if is_expert_weight: -- # We've checked that this is an expert weight -- # However it's not mapped locally to this rank -- # So we simply skip it -- continue -- -- # Skip loading extra bias for GPTQ models. -- if name.endswith(".bias") and name not in params_dict: -- continue -- -- remapped_name = maybe_remap_kv_scale_name(name, params_dict) -- if remapped_name is None: -- continue -- name = remapped_name -- -- # According to DeepSeek-V3 Technical Report, MTP modules -- # shares embedding layer. We only load the first weights. -- if ( -- spec_layer != self.model.mtp_start_layer_idx -- and ".layers" not in name -- ): -- continue -- -- param = params_dict[name] -- weight_loader = getattr( -- param, "weight_loader", default_weight_loader -- ) -- weight_loader(param, loaded_weight) -- if not is_fusion_moe_shared_experts_layer: -- loaded_params.add(name) -- -- # Validate that weights were loaded for each expected MTP layer. -- loaded_layers: set[int] = set() -- for param_name in loaded_params: -- spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name) -- if spec_layer is not None: -- loaded_layers.add(spec_layer) -- for layer_idx in range( -- self.model.mtp_start_layer_idx, -- self.model.mtp_start_layer_idx + self.model.num_mtp_layers, -- ): -- if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled(): -- raise ValueError( -- f"MTP speculative decoding layer {layer_idx} weights " -- f"missing from checkpoint. The checkpoint may have " -- f"been quantized without including the MTP layers. " -- f"Use a checkpoint that includes MTP layer weights, " -- f"or disable speculative decoding." -- ) -- -- return loaded_params -- -- def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: -- """ -- Rewrite the weight name to match the format of the original model. -- Add .mtp_block for modules in transformer layer block for spec layer -- and rename shared layer weights to be top level. -- """ -- spec_layer_weight_names = [ -- "embed_tokens", -- "enorm", -- "hnorm", -- "eh_proj", -- "shared_head", -- ] -- shared_weight_names = ["embed_tokens"] -- spec_layer_weight = False -- shared_weight = False -- for weight_name in spec_layer_weight_names: -- if weight_name in name: -- spec_layer_weight = True -- if weight_name in shared_weight_names: -- shared_weight = True -- break -- if not spec_layer_weight: -- # treat rest weights as weights for transformer layer block -- name = name.replace( -- f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block." -- ) -- elif shared_weight: -- # treat shared weights as top level weights -- name = name.replace(f"model.layers.{spec_layer}.", "model.") -- return name -+ -+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: -+ return self.norm(hidden_states) -+ -+ -+class DeepSeekMultiTokenPredictorLayer(nn.Module): -+ def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: -+ super().__init__() -+ -+ assert vllm_config.speculative_config is not None -+ config = vllm_config.speculative_config.draft_model_config.hf_config -+ self.config = config -+ quant_config = vllm_config.quant_config -+ -+ self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) -+ self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) -+ self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) -+ -+ self.device = current_platform.device_type -+ -+ self.is_v32 = hasattr(config, "index_topk") -+ if self.is_v32: -+ topk_tokens = config.index_topk -+ topk_indices_buffer = torch.empty( -+ vllm_config.scheduler_config.max_num_batched_tokens, -+ topk_tokens, -+ dtype=torch.int32, -+ device=self.device, -+ ) -+ else: -+ topk_indices_buffer = None -+ -+ self.shared_head = SharedHead( -+ config=config, prefix=prefix, quant_config=quant_config -+ ) -+ self.mtp_block = DeepseekV2DecoderLayer( -+ vllm_config, -+ prefix, -+ config=self.config, -+ topk_indices_buffer=topk_indices_buffer, -+ ) -+ -+ def forward( -+ self, -+ input_ids: torch.Tensor, -+ positions: torch.Tensor, -+ previous_hidden_states: torch.Tensor, -+ inputs_embeds: torch.Tensor | None = None, -+ spec_step_index: int = 0, -+ ) -> torch.Tensor: -+ assert inputs_embeds is not None -+ # masking inputs at position 0, as not needed by MTP -+ inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds) -+ inputs_embeds = self.enorm(inputs_embeds) -+ previous_hidden_states = self.hnorm(previous_hidden_states) -+ -+ hidden_states = self.eh_proj( -+ torch.cat([inputs_embeds, previous_hidden_states], dim=-1) -+ ) -+ -+ hidden_states, residual = self.mtp_block( -+ positions=positions, -+ hidden_states=hidden_states, -+ residual=None, -+ ) -+ hidden_states = residual + hidden_states # pre-final-norm (logits hidden) -+ if self.mtp_block.use_sequence_parallel_moe: -+ hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) -+ hidden_states = hidden_states[: positions.shape[0]] -+ # Recycle the post-final-norm hidden into the next draft step. -+ # compute_logits applies shared_head (== final norm) to the pre-norm -+ # element, so logits and the recycle each get exactly one final-norm. -+ # Matches SGLang's deepseek_nextn. -+ return hidden_states, self.shared_head(hidden_states) -+ -+ -+class DeepSeekMultiTokenPredictor(nn.Module): -+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): -+ super().__init__() -+ config = vllm_config.model_config.hf_config -+ self.mtp_start_layer_idx = config.num_hidden_layers -+ self.num_mtp_layers = config.num_nextn_predict_layers -+ # to map the exact layer index from weights -+ -+ self.layers = torch.nn.ModuleDict( -+ { -+ str(idx): DeepSeekMultiTokenPredictorLayer( -+ vllm_config, f"{prefix}.layers.{idx}" -+ ) -+ for idx in range( -+ self.mtp_start_layer_idx, -+ self.mtp_start_layer_idx + self.num_mtp_layers, -+ ) -+ } -+ ) -+ self.embed_tokens = VocabParallelEmbedding( -+ config.vocab_size, -+ config.hidden_size, -+ prefix=maybe_prefix(prefix, "embed_tokens"), -+ ) -+ self.logits_processor = LogitsProcessor(config.vocab_size) -+ -+ def set_skip_topk(self, skip: bool): -+ """Toggle skip_topk on all MTP layers with sparse attention. -+ -+ Called by the proposer to implement index_share_for_mtp_iteration: -+ step 0 sets skip=False (compute own indices), steps 1+ set skip=True -+ (reuse step 0's indices). -+ """ -+ for layer in self.layers.values(): -+ mtp_block = getattr(layer, "mtp_block", None) -+ if mtp_block is not None: -+ self_attn = getattr(mtp_block, "self_attn", None) -+ if self_attn is not None: -+ mla_attn = getattr(self_attn, "mla_attn", None) -+ if mla_attn is not None and hasattr(mla_attn, "skip_topk"): -+ mla_attn.skip_topk = skip -+ -+ def compact_topk_indices(self, slot_ids: torch.Tensor): -+ """Gather the top-k index rows at ``slot_ids`` to the front of the buffer.""" -+ num_slots = slot_ids.numel() -+ for layer in self.layers.values(): -+ mtp_block = getattr(layer, "mtp_block", None) -+ if mtp_block is not None: -+ self_attn = getattr(mtp_block, "self_attn", None) -+ if self_attn is not None: -+ mla_attn = getattr(self_attn, "mla_attn", None) -+ if mla_attn is not None and hasattr( -+ mla_attn, "topk_indices_buffer" -+ ): -+ topk_indices_buffer = mla_attn.topk_indices_buffer -+ topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] -+ -+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: -+ return self.embed_tokens(input_ids) -+ -+ def forward( -+ self, -+ input_ids: torch.Tensor, -+ positions: torch.Tensor, -+ previous_hidden_states: torch.Tensor, -+ inputs_embeds: torch.Tensor | None = None, -+ spec_step_idx: int = 0, -+ ) -> torch.Tensor: -+ if inputs_embeds is None: -+ inputs_embeds = self.embed_tokens(input_ids) -+ current_step_idx = spec_step_idx % self.num_mtp_layers -+ return self.layers[str(self.mtp_start_layer_idx + current_step_idx)]( -+ input_ids, -+ positions, -+ previous_hidden_states, -+ inputs_embeds, -+ current_step_idx, -+ ) -+ -+ def compute_logits( -+ self, -+ hidden_states: torch.Tensor, -+ spec_step_idx: int = 0, -+ ) -> torch.Tensor: -+ current_step_idx = spec_step_idx % self.num_mtp_layers -+ mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] -+ logits = self.logits_processor( -+ mtp_layer.shared_head.head, mtp_layer.shared_head(hidden_states) -+ ) -+ return logits -+ -+ -+@support_torch_compile -+class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): -+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): -+ super().__init__() -+ self.config = vllm_config.model_config.hf_config -+ self.quant_config = vllm_config.quant_config -+ self.model = DeepSeekMultiTokenPredictor( -+ vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") -+ ) -+ # Set MoE hyperparameters -+ self.set_moe_parameters() -+ -+ def set_moe_parameters(self): -+ self.num_moe_layers = self.config.num_nextn_predict_layers -+ self.num_expert_groups = self.config.n_group -+ -+ self.moe_layers = [] -+ self.moe_mlp_layers = [] -+ example_moe = None -+ for layer in self.model.layers.values(): -+ assert isinstance(layer, DeepSeekMultiTokenPredictorLayer) -+ layer = layer.mtp_block -+ assert isinstance(layer, DeepseekV2DecoderLayer) -+ if isinstance(layer.mlp, DeepseekV2MoE): -+ example_moe = layer.mlp -+ self.moe_mlp_layers.append(layer.mlp) -+ self.moe_layers.append(layer.mlp.experts) -+ self.extract_moe_parameters(example_moe) -+ self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( -+ self.model.layers.values(), -+ DeepseekV2MoE, -+ "mtp_block.mlp", -+ ) -+ -+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: -+ return self.model.embed_input_ids(input_ids) -+ -+ def forward( -+ self, -+ input_ids: torch.Tensor | None, -+ positions: torch.Tensor, -+ hidden_states: torch.Tensor, -+ intermediate_tensors: IntermediateTensors | None = None, -+ inputs_embeds: torch.Tensor | None = None, -+ spec_step_idx: int = 0, -+ ) -> torch.Tensor: -+ hidden_states = self.model( -+ input_ids, -+ positions, -+ hidden_states, -+ inputs_embeds, -+ spec_step_idx, -+ ) -+ return hidden_states -+ -+ def compute_logits( -+ self, -+ hidden_states: torch.Tensor, -+ spec_step_idx: int = 0, -+ ) -> torch.Tensor | None: -+ return self.model.compute_logits(hidden_states, spec_step_idx) -+ -+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: -+ stacked_params_mapping = [ -+ ("gate_up_proj", "gate_proj", 0), -+ ("gate_up_proj", "up_proj", 1), -+ ("fused_qkv_a_proj", "q_a_proj", 0), -+ ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), -+ ] -+ -+ # Fused indexer wk + weights_proj (shard 0 = wk, shard 1 = weights_proj) -+ indexer_fused_mapping = [ -+ ("wk_weights_proj", "wk", 0), -+ ("wk_weights_proj", "weights_proj", 1), -+ ] -+ stacked_params_mapping.extend(indexer_fused_mapping) -+ -+ expert_params_mapping = fused_moe_make_expert_params_mapping( -+ self, -+ ckpt_gate_proj_name="gate_proj", -+ ckpt_down_proj_name="down_proj", -+ ckpt_up_proj_name="up_proj", -+ num_experts=self.config.n_routed_experts -+ + ( -+ self.config.n_shared_experts -+ if self.is_fused_shared_expert_enabled -+ else 0 -+ ), -+ ) -+ -+ pp_missing_layer_names = get_pp_missing_layer_names(self) -+ params_dict = dict(self.named_parameters()) -+ loaded_params: set[str] = set() -+ _pending_wk_fp8: dict = {} # FP8 indexer wk dequant buffer -+ for name, loaded_weight in weights: -+ if "rotary_emb.inv_freq" in name: -+ continue -+ spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) -+ if spec_layer is None: -+ continue -+ is_fusion_moe_shared_experts_layer = ( -+ self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) -+ ) -+ name = self._rewrite_spec_layer_name(spec_layer, name) -+ -+ if _try_load_fp8_indexer_wk( -+ name, -+ loaded_weight, -+ _pending_wk_fp8, -+ params_dict, -+ loaded_params, -+ pp_missing_layer_names, + ): -+ continue -+ -+ for param_name, weight_name, shard_id in stacked_params_mapping: -+ # Skip non-stacked layers and experts (experts handled below). -+ if weight_name not in name: -+ continue -+ # We have mlp.experts[0].gate_proj in the checkpoint. -+ # Since we handle the experts below in expert_params_mapping, -+ # we need to skip here BEFORE we update the name, otherwise -+ # name will be updated to mlp.experts[0].gate_up_proj, which -+ # will then be updated below in expert_params_mapping -+ # for mlp.experts[0].gate_gate_up_proj, which breaks load. -+ if ("mlp.experts." in name) and name not in params_dict: -+ continue -+ if is_fusion_moe_shared_experts_layer: -+ continue -+ name_mapped = name.replace(weight_name, param_name) -+ -+ # QKV fusion is optional, fall back to normal -+ # weight loading if it's not enabled -+ if ( -+ param_name == "fused_qkv_a_proj" -+ ) and name_mapped not in params_dict: -+ continue -+ else: -+ name = name_mapped -+ -+ # Skip loading extra bias for GPTQ models. -+ if name.endswith(".bias") and name not in params_dict: -+ continue -+ -+ param = params_dict[name] -+ weight_loader = param.weight_loader -+ weight_loader(param, loaded_weight, shard_id) -+ break -+ else: -+ # Special handling: when AITER fusion_shared_experts is enabled, -+ # checkpoints may provide a single widened shared_experts tensor -+ # without explicit expert indices -+ # (e.g. ...mlp.shared_experts.gate_proj.weight). -+ # For models with multiple shared experts, split that tensor -+ # evenly into per-shared-expert slices and load them into -+ # appended expert slots mlp.experts.{n_routed_experts + j}.* -+ # accordingly. -+ num_chunks = 1 -+ if is_fusion_moe_shared_experts_layer: -+ num_chunks = getattr(self.config, "n_shared_experts", 1) or 1 -+ # Determine split axis based on op type -+ # gate/up: ColumnParallel → split along dim 0 -+ # down: RowParallel → split along dim 1 -+ split_dim = ( -+ 1 -+ if ("down_proj.weight" in name and loaded_weight.ndim > 1) -+ else 0 -+ ) -+ total = loaded_weight.shape[split_dim] -+ assert total % num_chunks == 0, ( -+ f"Shared expert weight dim {total} " -+ f"not divisible by num_chunks {num_chunks}" -+ ) -+ chunk_size = total // num_chunks -+ -+ for j in range(num_chunks): -+ chunk_name = name -+ weight_to_load = loaded_weight -+ -+ if is_fusion_moe_shared_experts_layer: -+ chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size) -+ if loaded_weight.ndim == 1: -+ weight_to_load = loaded_weight[chunk_slice] -+ elif split_dim == 0: -+ weight_to_load = loaded_weight[chunk_slice, :] -+ else: -+ weight_to_load = loaded_weight[:, chunk_slice] -+ # Synthesize an expert-style name so expert mapping -+ # can route it -+ chunk_name = name.replace( -+ "mlp.shared_experts", -+ f"mlp.experts.{self.config.n_routed_experts + j}", -+ ) -+ -+ # Use expert_params_mapping to locate the destination -+ # param and delegate to its expert-aware weight_loader -+ # with expert_id. -+ is_expert_weight = False -+ for mapping in expert_params_mapping: -+ param_name, weight_name, expert_id, expert_shard_id = mapping -+ if weight_name not in chunk_name: -+ continue -+ -+ # Anyway, this is an expert weight and should not be -+ # attempted to load as other weights later -+ is_expert_weight = True -+ -+ # Do not modify `name` since the loop may continue here -+ # Instead, create a new variable -+ name_mapped = chunk_name.replace(weight_name, param_name) -+ -+ param = params_dict[name_mapped] -+ # We should ask the weight loader to return success or -+ # not here since otherwise we may skip experts with -+ # other available replicas. -+ weight_loader = typing.cast( -+ Callable[..., bool], param.weight_loader -+ ) -+ success = weight_loader( -+ param, -+ weight_to_load, -+ name_mapped, -+ shard_id=expert_shard_id, -+ expert_id=expert_id, -+ return_success=True, -+ ) -+ if success: -+ if not is_fusion_moe_shared_experts_layer: -+ name = name_mapped -+ else: -+ loaded_params.add(name_mapped) -+ break -+ else: -+ if is_expert_weight: -+ # We've checked that this is an expert weight -+ # However it's not mapped locally to this rank -+ # So we simply skip it -+ continue -+ -+ # Skip loading extra bias for GPTQ models. -+ if name.endswith(".bias") and name not in params_dict: -+ continue -+ -+ remapped_name = maybe_remap_kv_scale_name(name, params_dict) -+ if remapped_name is None: -+ continue -+ name = remapped_name -+ -+ # According to DeepSeek-V3 Technical Report, MTP modules -+ # shares embedding layer. We only load the first weights. -+ if ( -+ spec_layer != self.model.mtp_start_layer_idx -+ and ".layers" not in name -+ ): -+ continue -+ -+ param = params_dict[name] -+ weight_loader = getattr( -+ param, "weight_loader", default_weight_loader -+ ) -+ weight_loader(param, loaded_weight) -+ if not is_fusion_moe_shared_experts_layer: -+ loaded_params.add(name) -+ -+ # Validate that weights were loaded for each expected MTP layer. -+ loaded_layers: set[int] = set() -+ for param_name in loaded_params: -+ spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name) -+ if spec_layer is not None: -+ loaded_layers.add(spec_layer) -+ for layer_idx in range( -+ self.model.mtp_start_layer_idx, -+ self.model.mtp_start_layer_idx + self.model.num_mtp_layers, -+ ): -+ if layer_idx not in loaded_layers and is_mtp_completeness_check_enabled(): + raise ValueError( -+ f"MTP speculative decoding layer {layer_idx} weights " -+ f"missing from checkpoint. The checkpoint may have " -+ f"been quantized without including the MTP layers. " -+ f"Use a checkpoint that includes MTP layer weights, " -+ f"or disable speculative decoding." ++ "Runtime NVFP4 proposal-head quantization requires an " ++ "unquantized checkpoint head" + ) -+ -+ return loaded_params -+ -+ def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: -+ """ -+ Rewrite the weight name to match the format of the original model. -+ Add .mtp_block for modules in transformer layer block for spec layer -+ and rename shared layer weights to be top level. -+ """ -+ spec_layer_weight_names = [ -+ "embed_tokens", -+ "enorm", -+ "hnorm", -+ "eh_proj", -+ "shared_head", -+ ] -+ shared_weight_names = ["embed_tokens"] -+ spec_layer_weight = False -+ shared_weight = False -+ for weight_name in spec_layer_weight_names: -+ if weight_name in name: -+ spec_layer_weight = True -+ if weight_name in shared_weight_names: -+ shared_weight = True -+ break -+ if not spec_layer_weight: -+ # treat rest weights as weights for transformer layer block -+ name = name.replace( -+ f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block." ++ quant_method = Nvfp4OnlineLinearMethod( ++ use_a16=envs.VLLM_LM_HEAD_A16 + ) -+ elif shared_weight: -+ # treat shared weights as top level weights -+ name = name.replace(f"model.layers.{spec_layer}.", "model.") -+ return name -diff --git a/vllm/models/glm5next/model_state.py b/vllm/models/glm5next/model_state.py -index edeae144563c42df7bf55b7147ea7fdd3951fbf4..316332cb3f695d76705743d9b44629680fe582ab 100644 ++ self.runtime_lm_head_quantization = "nvfp4" + quant_method_implements_embedding = method_has_implemented_embedding( + type(quant_method) + ) +@@ -549,6 +570,7 @@ + prefix: str = "", + *, + disable_tp: bool = False, ++ lm_head_quantization: Literal["nvfp4"] | None = None, + ): + super().__init__( + num_embeddings, +@@ -559,6 +581,7 @@ + quant_config, + prefix, + disable_tp=disable_tp, ++ lm_head_quantization=lm_head_quantization, + ) + self.quant_config = quant_config + if bias: +--- a/vllm/model_executor/models/deepseek_mtp.py ++++ b/vllm/model_executor/models/deepseek_mtp.py +@@ -52,6 +52,8 @@ + config: PretrainedConfig, + prefix: str, + quant_config: QuantizationConfig | None = None, ++ *, ++ lm_head_quantization: typing.Literal["nvfp4"] | None = None, + ) -> None: + super().__init__() + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) +@@ -60,6 +62,7 @@ + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "head"), ++ lm_head_quantization=lm_head_quantization, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: --- a/vllm/models/glm5next/model_state.py +++ b/vllm/models/glm5next/model_state.py -@@ -310,25 +310,13 @@ class Glm5NextModelState(MambaHybridModelState): +@@ -310,25 +310,13 @@ num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np) if self._align_mode: @@ -3104,495 +561,49 @@ index edeae144563c42df7bf55b7147ea7fdd3951fbf4..316332cb3f695d76705743d9b4462968 model_metadata = Glm5NextAttnMetadata( is_prefilling=is_prefilling, -diff --git a/vllm/models/glm5next/nvidia/mtp.py b/vllm/models/glm5next/nvidia/mtp.py -index 90bdb5b642144d2e5a29b9d2e7cb8d8c9277c71c..407ed1f21a72923fd000d564ab7d66f26015d538 100644 +--- a/vllm/models/glm5next/nvidia/model.py ++++ b/vllm/models/glm5next/nvidia/model.py +@@ -1334,7 +1334,7 @@ + return False + + entry = buf.setdefault(layer_prefix, {}).setdefault(key, {}) +- entry["weight" if is_weight else "scale"] = tensor ++ entry["weight" if is_weight else "scale"] = tensor.clone() + if "weight" not in entry or "scale" not in entry: + return True + +@@ -1385,7 +1385,7 @@ + return False + + entry = buf.setdefault(layer_prefix, {}).setdefault("indexer_weights", {}) +- entry["weight" if is_weight else "scale"] = tensor ++ entry["weight" if is_weight else "scale"] = tensor.clone() + if "weight" not in entry or "scale" not in entry: + return True + --- a/vllm/models/glm5next/nvidia/mtp.py +++ b/vllm/models/glm5next/nvidia/mtp.py -@@ -1,475 +1,515 @@ --# SPDX-License-Identifier: Apache-2.0 --# SPDX-FileCopyrightText: Copyright contributors to the vLLM project --import typing --from collections.abc import Callable, Iterable -- -+# SPDX-License-Identifier: Apache-2.0 -+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -+import typing -+from collections.abc import Callable, Iterable -+ +@@ -6,6 +6,7 @@ import torch import torch.nn as nn +import vllm.envs as envs from vllm.config import VllmConfig --from vllm.model_executor.layers.fused_moe import ( -- fused_moe_make_expert_params_mapping, --) --from vllm.model_executor.layers.layernorm import RMSNorm --from vllm.model_executor.layers.logits_processor import LogitsProcessor --from vllm.model_executor.layers.vocab_parallel_embedding import ( -- VocabParallelEmbedding, --) --from vllm.model_executor.model_loader.weight_utils import ( -- default_weight_loader, -- maybe_remap_kv_scale_name, --) --from vllm.model_executor.models.deepseek_mtp import SharedHead --from vllm.model_executor.models.deepseek_v2 import DeepseekV2MixtureOfExperts --from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix --from vllm.platforms import current_platform --from vllm.sequence import IntermediateTensors -- --from .model import ( -- GLM5NEXT_PACKED_MODULES_MAPPING, -- Glm5NextDecoderLayer, -- Glm5NextMoE, -- _try_load_fp8_attn_proj, -- _try_load_mxfp8_bf16_attn_proj, -- get_spec_layer_idx_from_weight_name, --) --from .pooled_indexer import Glm5NextPooledIndexer -- -- --class Glm5NextMultiTokenPredictorLayer(nn.Module): -- def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: -- super().__init__() -- assert vllm_config.speculative_config is not None -- config = vllm_config.speculative_config.draft_model_config.hf_config -- self.config = config -- quant_config = vllm_config.quant_config -- -- self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) -- self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) -- self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) -- -- topk_tokens = config.index_topk -- kpool = getattr(config, "index_kpool", 1) or 1 -- buffer_width = topk_tokens + (kpool - 1 if kpool > 1 else 0) -- topk_indices_buffer = torch.empty( -- vllm_config.scheduler_config.max_num_batched_tokens, -- buffer_width, -- dtype=torch.int32, -- device=current_platform.device_type, -- ) -- pool_topk_indices_buffer = torch.empty( -- vllm_config.scheduler_config.max_num_batched_tokens, -- topk_tokens // kpool, -- dtype=torch.int32, -- device=current_platform.device_type, -- ) -+from vllm.model_executor.layers.fused_moe import ( -+ fused_moe_make_expert_params_mapping, -+) -+from vllm.model_executor.layers.layernorm import RMSNorm -+from vllm.model_executor.layers.logits_processor import LogitsProcessor -+from vllm.model_executor.layers.vocab_parallel_embedding import ( -+ VocabParallelEmbedding, -+) -+from vllm.model_executor.model_loader.weight_utils import ( -+ default_weight_loader, -+ maybe_remap_kv_scale_name, -+) -+from vllm.model_executor.models.deepseek_mtp import SharedHead -+from vllm.model_executor.models.deepseek_v2 import DeepseekV2MixtureOfExperts -+from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix -+from vllm.platforms import current_platform -+from vllm.sequence import IntermediateTensors -+ -+from .model import ( -+ GLM5NEXT_PACKED_MODULES_MAPPING, -+ Glm5NextDecoderLayer, -+ Glm5NextMoE, -+ _try_load_fp8_attn_proj, -+ _try_load_mxfp8_bf16_attn_proj, -+ get_spec_layer_idx_from_weight_name, -+) -+from .pooled_indexer import Glm5NextPooledIndexer -+ -+ -+class Glm5NextMultiTokenPredictorLayer(nn.Module): -+ def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: -+ super().__init__() -+ assert vllm_config.speculative_config is not None -+ config = vllm_config.speculative_config.draft_model_config.hf_config -+ self.config = config -+ quant_config = vllm_config.quant_config -+ -+ self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) -+ self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) -+ self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) -+ -+ topk_tokens = config.index_topk -+ kpool = getattr(config, "index_kpool", 1) or 1 -+ buffer_width = topk_tokens + (kpool - 1 if kpool > 1 else 0) -+ topk_indices_buffer = torch.empty( -+ vllm_config.scheduler_config.max_num_batched_tokens, -+ buffer_width, -+ dtype=torch.int32, -+ device=current_platform.device_type, -+ ) -+ pool_topk_indices_buffer = torch.empty( -+ vllm_config.scheduler_config.max_num_batched_tokens, -+ topk_tokens // kpool, -+ dtype=torch.int32, -+ device=current_platform.device_type, -+ ) + from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +@@ -64,7 +65,10 @@ + device=current_platform.device_type, + ) self.shared_head = SharedHead( - config=config, prefix=prefix, quant_config=quant_config -- ) -- # MTP layers sit past the base model's hidden layers; parse the index -- # from the prefix (e.g. "...layers.32") so the decoder builds an MLA -- # (DSA) layer rather than KDA for the MTP path. -- layer_idx = int(prefix.rsplit(".", 1)[-1]) -- self.mtp_block = Glm5NextDecoderLayer( -- vllm_config=vllm_config, - config=config, -- layer_idx=layer_idx, - prefix=prefix, -- topk_indices_buffer=topk_indices_buffer, -- pool_topk_indices_buffer=pool_topk_indices_buffer, -- is_mtp_layer=True, -- ) -- -- def forward( -- self, -- input_ids: torch.Tensor, -- positions: torch.Tensor, -- previous_hidden_states: torch.Tensor, -- inputs_embeds: torch.Tensor | None = None, -- spec_step_index: int = 0, -- output_indices: torch.Tensor | None = None, -- ) -> torch.Tensor: -- assert inputs_embeds is not None -- eh_input = torch.cat( -- (self.enorm(inputs_embeds), self.hnorm(previous_hidden_states)), -- dim=-1, ++ config=config, ++ prefix=prefix, + quant_config=quant_config, + lm_head_quantization="nvfp4" if envs.VLLM_MTP_NVFP4_LM_HEAD else None, ) -- hidden_states = self.eh_proj(eh_input) -- # Fuse the residual add and final RMSNorm. Glm5NextMoE already performs -- # its all-reduce, so no collective is needed here. The post-norm result -- # feeds both draft logits and the next recycled hidden state. -- hidden_states, residual, _, _ = self.mtp_block( -- positions=positions, -- hidden_states=hidden_states, -- residual=None, -- output_indices=output_indices, -- ) -- hidden_states, _ = self.shared_head.norm(hidden_states, residual=residual) -- return hidden_states, hidden_states -- -- --class Glm5NextMultiTokenPredictor(nn.Module): -- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): -- super().__init__() -- config = vllm_config.model_config.hf_config -- self.mtp_start_layer_idx = config.num_hidden_layers -- self.num_mtp_layers = config.num_nextn_predict_layers -- self.layers = torch.nn.ModuleDict( -- { -- str(idx): Glm5NextMultiTokenPredictorLayer( -- vllm_config, f"{prefix}.layers.{idx}" -- ) -- for idx in range( -- self.mtp_start_layer_idx, -- self.mtp_start_layer_idx + self.num_mtp_layers, -- ) -- } -- ) -- self.embed_tokens = VocabParallelEmbedding( -- config.vocab_size, -- config.hidden_size, -- prefix=maybe_prefix(prefix, "embed_tokens"), -- ) -- # Plain list for the per-propose lookup: ModuleDict[str(...)] builds a -- # string and hashes it on every draft step. -- self._mtp_layers = list(self.layers.values()) -- self._prefill_output_indices: torch.Tensor | None = None -- self.logits_processor = LogitsProcessor(config.vocab_size) -- -- def update_max_model_len(self, max_model_len: int) -> None: -- for module in self.modules(): -- if isinstance(module, Glm5NextPooledIndexer): -- module.update_max_model_len(max_model_len) -- -- def set_skip_topk(self, skip: bool): -- # index_share_for_mtp_iteration: step 0 computes top-k, steps 1+ reuse. -- for layer in self.layers.values(): -- self_attn = getattr(layer.mtp_block, "self_attn", None) -- mla_attn = getattr(self_attn, "mla_attn", None) -- if mla_attn is not None and hasattr(mla_attn, "skip_topk"): -- mla_attn.skip_topk = skip -- -- def compact_topk_indices(self, slot_ids: torch.Tensor): -- """Gather the top-k index rows at ``slot_ids`` to the front of the buffer.""" -- num_slots = slot_ids.numel() -- for layer in self.layers.values(): -- self_attn = getattr(layer.mtp_block, "self_attn", None) -- mla_attn = getattr(self_attn, "mla_attn", None) -- if mla_attn is not None and hasattr(mla_attn, "topk_indices_buffer"): -- topk_indices_buffer = mla_attn.topk_indices_buffer -- topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] -- -- def snapshot_qsa_interval_starts(self) -> None: -- for layer in self.layers.values(): -- self_attn = getattr(layer.mtp_block, "self_attn", None) -- indexer = getattr(self_attn, "indexer", None) -- snapshot = getattr(indexer, "snapshot_speculative_interval_starts", None) -- if snapshot is not None: -- snapshot() -- -- def restore_qsa_interval_starts(self) -> None: -- for layer in self.layers.values(): -- self_attn = getattr(layer.mtp_block, "self_attn", None) -- indexer = getattr(self_attn, "indexer", None) -- restore = getattr(indexer, "restore_speculative_interval_starts", None) -- if restore is not None: -- restore() -- -- def set_prefill_output_indices(self, output_indices: torch.Tensor | None) -> None: -- """Select request-tail outputs after populating all MTP attention caches.""" -- self._prefill_output_indices = output_indices -- -- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: -- return self.embed_tokens(input_ids) -- -- def forward( -- self, -- input_ids: torch.Tensor, -- positions: torch.Tensor, -- previous_hidden_states: torch.Tensor, -- inputs_embeds: torch.Tensor | None = None, -- spec_step_idx: int = 0, -- ) -> torch.Tensor: -- if inputs_embeds is None: -- inputs_embeds = self.embed_tokens(input_ids) -- current_step_idx = spec_step_idx % self.num_mtp_layers -- return self._mtp_layers[current_step_idx]( -- input_ids, -- positions, -- previous_hidden_states, -- inputs_embeds, -- current_step_idx, -- self._prefill_output_indices, -- ) -- -- def compute_logits( -- self, -- hidden_states: torch.Tensor, -- spec_step_idx: int = 0, -- ) -> torch.Tensor: -- current_step_idx = spec_step_idx % self.num_mtp_layers -- mtp_layer = self._mtp_layers[current_step_idx] -- # hidden_states is already post-final-norm (produced in the layer -- # forward and recycled as-is); apply the LM head only, without a -- # second RMSNorm. -- return self.logits_processor(mtp_layer.shared_head.head, hidden_states) -- -- def get_top_tokens( -- self, -- hidden_states: torch.Tensor, -- spec_step_idx: int = 0, -- ) -> torch.Tensor: -- current_step_idx = spec_step_idx % self.num_mtp_layers -- mtp_layer = self._mtp_layers[current_step_idx] -- # Vocab-parallel argmax for the greedy draft: per-rank head projection -- # + local argmax + a [batch, 2*tp] (value, index) reduce, instead of -- # materializing and all-gathering full [N, vocab] logits per draft -- # step. Tie-breaking matches the full argmax (shards are contiguous -- # and rank-ordered, so the lowest-rank winner is the lowest global -- # index), so greedy draft tokens are unchanged. -- return self.logits_processor.get_top_tokens( -- mtp_layer.shared_head.head, hidden_states -- ) -- -- --class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts): -- packed_modules_mapping = GLM5NEXT_PACKED_MODULES_MAPPING -- hf_to_vllm_mapper = WeightsMapper( -- orig_to_new_prefix={ -- "model.language_model.": "model.", -- "language_model.model.": "model.", -- } -- ) -- -- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): -+ # MTP layers sit past the base model's hidden layers; parse the index -+ # from the prefix (e.g. "...layers.32") so the decoder builds an MLA -+ # (DSA) layer rather than KDA for the MTP path. -+ layer_idx = int(prefix.rsplit(".", 1)[-1]) -+ self.mtp_block = Glm5NextDecoderLayer( -+ vllm_config=vllm_config, -+ config=config, -+ layer_idx=layer_idx, -+ prefix=prefix, -+ topk_indices_buffer=topk_indices_buffer, -+ pool_topk_indices_buffer=pool_topk_indices_buffer, -+ is_mtp_layer=True, -+ ) -+ -+ def forward( -+ self, -+ input_ids: torch.Tensor, -+ positions: torch.Tensor, -+ previous_hidden_states: torch.Tensor, -+ inputs_embeds: torch.Tensor | None = None, -+ spec_step_index: int = 0, -+ output_indices: torch.Tensor | None = None, -+ ) -> torch.Tensor: -+ assert inputs_embeds is not None -+ eh_input = torch.cat( -+ (self.enorm(inputs_embeds), self.hnorm(previous_hidden_states)), -+ dim=-1, -+ ) -+ hidden_states = self.eh_proj(eh_input) -+ # Fuse the residual add and final RMSNorm. Glm5NextMoE already performs -+ # its all-reduce, so no collective is needed here. The post-norm result -+ # feeds both draft logits and the next recycled hidden state. -+ hidden_states, residual, _, _ = self.mtp_block( -+ positions=positions, -+ hidden_states=hidden_states, -+ residual=None, -+ output_indices=output_indices, -+ ) -+ hidden_states, _ = self.shared_head.norm(hidden_states, residual=residual) -+ return hidden_states, hidden_states -+ -+ -+class Glm5NextMultiTokenPredictor(nn.Module): -+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): -+ super().__init__() -+ config = vllm_config.model_config.hf_config -+ self.mtp_start_layer_idx = config.num_hidden_layers -+ self.num_mtp_layers = config.num_nextn_predict_layers -+ self.layers = torch.nn.ModuleDict( -+ { -+ str(idx): Glm5NextMultiTokenPredictorLayer( -+ vllm_config, f"{prefix}.layers.{idx}" -+ ) -+ for idx in range( -+ self.mtp_start_layer_idx, -+ self.mtp_start_layer_idx + self.num_mtp_layers, -+ ) -+ } -+ ) -+ self.embed_tokens = VocabParallelEmbedding( -+ config.vocab_size, -+ config.hidden_size, -+ prefix=maybe_prefix(prefix, "embed_tokens"), -+ ) -+ # Plain list for the per-propose lookup: ModuleDict[str(...)] builds a -+ # string and hashes it on every draft step. -+ self._mtp_layers = list(self.layers.values()) -+ self._prefill_output_indices: torch.Tensor | None = None -+ self.logits_processor = LogitsProcessor(config.vocab_size) -+ -+ def update_max_model_len(self, max_model_len: int) -> None: -+ for module in self.modules(): -+ if isinstance(module, Glm5NextPooledIndexer): -+ module.update_max_model_len(max_model_len) -+ -+ def set_skip_topk(self, skip: bool): -+ # index_share_for_mtp_iteration: step 0 computes top-k, steps 1+ reuse. -+ for layer in self.layers.values(): -+ self_attn = getattr(layer.mtp_block, "self_attn", None) -+ mla_attn = getattr(self_attn, "mla_attn", None) -+ if mla_attn is not None and hasattr(mla_attn, "skip_topk"): -+ mla_attn.skip_topk = skip -+ -+ def compact_topk_indices(self, slot_ids: torch.Tensor): -+ """Gather the top-k index rows at ``slot_ids`` to the front of the buffer.""" -+ num_slots = slot_ids.numel() -+ for layer in self.layers.values(): -+ self_attn = getattr(layer.mtp_block, "self_attn", None) -+ mla_attn = getattr(self_attn, "mla_attn", None) -+ if mla_attn is not None and hasattr(mla_attn, "topk_indices_buffer"): -+ topk_indices_buffer = mla_attn.topk_indices_buffer -+ topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] -+ -+ def snapshot_qsa_interval_starts(self) -> None: -+ for layer in self.layers.values(): -+ self_attn = getattr(layer.mtp_block, "self_attn", None) -+ indexer = getattr(self_attn, "indexer", None) -+ snapshot = getattr(indexer, "snapshot_speculative_interval_starts", None) -+ if snapshot is not None: -+ snapshot() -+ -+ def restore_qsa_interval_starts(self) -> None: -+ for layer in self.layers.values(): -+ self_attn = getattr(layer.mtp_block, "self_attn", None) -+ indexer = getattr(self_attn, "indexer", None) -+ restore = getattr(indexer, "restore_speculative_interval_starts", None) -+ if restore is not None: -+ restore() -+ -+ def set_prefill_output_indices(self, output_indices: torch.Tensor | None) -> None: -+ """Select request-tail outputs after populating all MTP attention caches.""" -+ self._prefill_output_indices = output_indices -+ -+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: -+ return self.embed_tokens(input_ids) -+ -+ def forward( -+ self, -+ input_ids: torch.Tensor, -+ positions: torch.Tensor, -+ previous_hidden_states: torch.Tensor, -+ inputs_embeds: torch.Tensor | None = None, -+ spec_step_idx: int = 0, -+ ) -> torch.Tensor: -+ if inputs_embeds is None: -+ inputs_embeds = self.embed_tokens(input_ids) -+ current_step_idx = spec_step_idx % self.num_mtp_layers -+ return self._mtp_layers[current_step_idx]( -+ input_ids, -+ positions, -+ previous_hidden_states, -+ inputs_embeds, -+ current_step_idx, -+ self._prefill_output_indices, -+ ) -+ -+ def compute_logits( -+ self, -+ hidden_states: torch.Tensor, -+ spec_step_idx: int = 0, -+ ) -> torch.Tensor: -+ current_step_idx = spec_step_idx % self.num_mtp_layers -+ mtp_layer = self._mtp_layers[current_step_idx] -+ # hidden_states is already post-final-norm (produced in the layer -+ # forward and recycled as-is); apply the LM head only, without a -+ # second RMSNorm. -+ return self.logits_processor(mtp_layer.shared_head.head, hidden_states) -+ -+ def get_top_tokens( -+ self, -+ hidden_states: torch.Tensor, -+ spec_step_idx: int = 0, -+ ) -> torch.Tensor: -+ current_step_idx = spec_step_idx % self.num_mtp_layers -+ mtp_layer = self._mtp_layers[current_step_idx] -+ # Vocab-parallel argmax for the greedy draft: per-rank head projection -+ # + local argmax + a [batch, 2*tp] (value, index) reduce, instead of -+ # materializing and all-gathering full [N, vocab] logits per draft -+ # step. Tie-breaking matches the full argmax (shards are contiguous -+ # and rank-ordered, so the lowest-rank winner is the lowest global -+ # index), so greedy draft tokens are unchanged. -+ return self.logits_processor.get_top_tokens( -+ mtp_layer.shared_head.head, hidden_states -+ ) -+ -+ -+class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts): -+ packed_modules_mapping = GLM5NEXT_PACKED_MODULES_MAPPING -+ hf_to_vllm_mapper = WeightsMapper( -+ orig_to_new_prefix={ -+ "model.language_model.": "model.", -+ "language_model.model.": "model.", -+ } -+ ) -+ -+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + # MTP layers sit past the base model's hidden layers; parse the index + # from the prefix (e.g. "...layers.32") so the decoder builds an MLA +@@ -245,14 +249,20 @@ super().__init__() self.config = vllm_config.model_config.hf_config self.quant_config = vllm_config.quant_config @@ -3608,267 +619,37 @@ index 90bdb5b642144d2e5a29b9d2e7cb8d8c9277c71c..407ed1f21a72923fd000d564ab7d66f2 + if self.has_own_lm_head: + self.lm_head = head self.set_moe_parameters() -- -- def _checkpoint_weight_name_prefixes(self) -> tuple[str, ...]: + + def _checkpoint_weight_name_prefixes(self) -> tuple[str, ...]: - return tuple( -+ -+ def _checkpoint_weight_name_prefixes(self) -> tuple[str, ...]: + prefixes = tuple( prefix -- for layer_idx in range( -- self.config.num_hidden_layers, -- self.config.num_hidden_layers + self.config.num_nextn_predict_layers, -- ) -- for prefix in ( -- f"model.language_model.layers.{layer_idx}.", -- f"language_model.model.layers.{layer_idx}.", -- f"model.layers.{layer_idx}.", -+ for layer_idx in range( -+ self.config.num_hidden_layers, -+ self.config.num_hidden_layers + self.config.num_nextn_predict_layers, -+ ) -+ for prefix in ( -+ f"model.language_model.layers.{layer_idx}.", -+ f"language_model.model.layers.{layer_idx}.", -+ f"model.layers.{layer_idx}.", + for layer_idx in range( + self.config.num_hidden_layers, +@@ -265,6 +275,14 @@ f"layers.{layer_idx}.", ) ) -- -- def set_moe_parameters(self): -- self.num_moe_layers = self.config.num_nextn_predict_layers -- self.num_expert_groups = self.config.n_group -- self.moe_layers = [] -- self.moe_mlp_layers = [] -- example_moe = None -- for layer in self.model.layers.values(): -- mlp = layer.mtp_block.mlp -- if isinstance(mlp, Glm5NextMoE): -- example_moe = mlp -- self.moe_mlp_layers.append(mlp) -- self.moe_layers.append(mlp.experts) -- self.extract_moe_parameters(example_moe) -- -- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: -- return self.model.embed_input_ids(input_ids) -- -- def update_max_model_len(self, max_model_len: int) -> None: -- self.model.update_max_model_len(max_model_len) -- -- def forward( -- self, -- input_ids: torch.Tensor | None, -- positions: torch.Tensor, -- hidden_states: torch.Tensor, -- intermediate_tensors: IntermediateTensors | None = None, -- inputs_embeds: torch.Tensor | None = None, -- spec_step_idx: int = 0, -- ) -> torch.Tensor: -- return self.model( -- input_ids, positions, hidden_states, inputs_embeds, spec_step_idx -- ) -- -- def compute_logits( -- self, -- hidden_states: torch.Tensor, -- spec_step_idx: int = 0, -- ) -> torch.Tensor | None: -- return self.model.compute_logits(hidden_states, spec_step_idx) -- -- def get_top_tokens( -- self, -- hidden_states: torch.Tensor, -- spec_step_idx: int = 0, -- ) -> torch.Tensor: -- # Greedy-draft path used when use_local_argmax_reduction is enabled: -- # vocab-parallel argmax, no full-vocab logits. -- return self.model.get_top_tokens(hidden_states, spec_step_idx) -- -- def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: -- spec_layer_weight_names = [ -- "embed_tokens", -- "enorm", -- "hnorm", -- "eh_proj", -- "shared_head", -- ] -- shared_weight_names = ["embed_tokens"] -- spec_layer_weight = False -- shared_weight = False -- for weight_name in spec_layer_weight_names: -- if weight_name in name: -- spec_layer_weight = True -- if weight_name in shared_weight_names: -- shared_weight = True -- break -- if not spec_layer_weight: -- name = name.replace( -- f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block." + if self.has_own_lm_head: + prefixes += ( + "lm_head.", + "model.lm_head.", + "model.language_model.lm_head.", + "language_model.lm_head.", - ) -- elif shared_weight: -- name = name.replace(f"model.layers.{spec_layer}.", "model.") -- return name -- -- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: -- stacked_params_mapping = [ -- ("gate_up_proj", "gate_proj", 0), -- ("gate_up_proj", "up_proj", 1), -- ("fused_qkv_a_proj", "q_a_proj", 0), -- ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), -- ] -- expert_params_mapping = fused_moe_make_expert_params_mapping( -- self, -- ckpt_gate_proj_name="gate_proj", -- ckpt_down_proj_name="down_proj", -- ckpt_up_proj_name="up_proj", -- num_experts=self.config.n_routed_experts, -- ) -- -- params_dict = dict(self.named_parameters()) -- loaded_params: set[str] = set() -- pending_attn_weights: dict = {} -- # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from -- # ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model. -- kv_a_pad_size = 0 -- if self.config.mla_nope and self.config.qk_rope_head_dim > 0: -- kv_a_pad_size = self.config.qk_rope_head_dim -- for name, loaded_weight in weights: -- if "rotary_emb.inv_freq" in name: -- continue -- # Multimodal (Glm5NextForConditionalGeneration) checkpoints prefix -- # the text-tower weights with "model.language_model."; the MTP head -- # is built as a text-only model (model.layers.*), so strip the -- # prefix to match. -+ return prefixes -+ -+ def set_moe_parameters(self): -+ self.num_moe_layers = self.config.num_nextn_predict_layers -+ self.num_expert_groups = self.config.n_group -+ self.moe_layers = [] -+ self.moe_mlp_layers = [] -+ example_moe = None -+ for layer in self.model.layers.values(): -+ mlp = layer.mtp_block.mlp -+ if isinstance(mlp, Glm5NextMoE): -+ example_moe = mlp -+ self.moe_mlp_layers.append(mlp) -+ self.moe_layers.append(mlp.experts) -+ self.extract_moe_parameters(example_moe) -+ -+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: -+ return self.model.embed_input_ids(input_ids) -+ -+ def update_max_model_len(self, max_model_len: int) -> None: -+ self.model.update_max_model_len(max_model_len) -+ -+ def forward( -+ self, -+ input_ids: torch.Tensor | None, -+ positions: torch.Tensor, -+ hidden_states: torch.Tensor, -+ intermediate_tensors: IntermediateTensors | None = None, -+ inputs_embeds: torch.Tensor | None = None, -+ spec_step_idx: int = 0, -+ ) -> torch.Tensor: -+ return self.model( -+ input_ids, positions, hidden_states, inputs_embeds, spec_step_idx -+ ) -+ -+ def compute_logits( -+ self, -+ hidden_states: torch.Tensor, -+ spec_step_idx: int = 0, -+ ) -> torch.Tensor | None: -+ return self.model.compute_logits(hidden_states, spec_step_idx) -+ -+ def get_top_tokens( -+ self, -+ hidden_states: torch.Tensor, -+ spec_step_idx: int = 0, -+ ) -> torch.Tensor: -+ # Greedy-draft path used when use_local_argmax_reduction is enabled: -+ # vocab-parallel argmax, no full-vocab logits. -+ return self.model.get_top_tokens(hidden_states, spec_step_idx) -+ -+ def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: -+ spec_layer_weight_names = [ -+ "embed_tokens", -+ "enorm", -+ "hnorm", -+ "eh_proj", -+ "shared_head", -+ ] -+ shared_weight_names = ["embed_tokens"] -+ spec_layer_weight = False -+ shared_weight = False -+ for weight_name in spec_layer_weight_names: -+ if weight_name in name: -+ spec_layer_weight = True -+ if weight_name in shared_weight_names: -+ shared_weight = True -+ break -+ if not spec_layer_weight: -+ name = name.replace( -+ f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block." + ) -+ elif shared_weight: -+ name = name.replace(f"model.layers.{spec_layer}.", "model.") -+ return name -+ -+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: -+ stacked_params_mapping = [ -+ ("gate_up_proj", "gate_proj", 0), -+ ("gate_up_proj", "up_proj", 1), -+ ("fused_qkv_a_proj", "q_a_proj", 0), -+ ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), -+ ] -+ expert_params_mapping = fused_moe_make_expert_params_mapping( -+ self, -+ ckpt_gate_proj_name="gate_proj", -+ ckpt_down_proj_name="down_proj", -+ ckpt_up_proj_name="up_proj", -+ num_experts=self.config.n_routed_experts, -+ ) -+ -+ params_dict = dict(self.named_parameters()) -+ loaded_params: set[str] = set() -+ pending_attn_weights: dict = {} -+ # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from -+ # ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model. -+ kv_a_pad_size = 0 -+ if self.config.mla_nope and self.config.qk_rope_head_dim > 0: -+ kv_a_pad_size = self.config.qk_rope_head_dim -+ for name, loaded_weight in weights: -+ if "rotary_emb.inv_freq" in name: -+ continue -+ # Multimodal (Glm5NextForConditionalGeneration) checkpoints prefix -+ # the text-tower weights with "model.language_model."; the MTP head -+ # is built as a text-only model (model.layers.*), so strip the -+ # prefix to match. ++ return prefixes + + def set_moe_parameters(self): + self.num_moe_layers = self.config.num_nextn_predict_layers +@@ -372,6 +390,19 @@ + # prefix to match. if name.startswith("model.language_model."): name = name.replace("model.language_model.", "model.", 1) -- spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) -- if spec_layer is None: -- continue -- name = self._rewrite_spec_layer_name(spec_layer, name) -- -- if _try_load_mxfp8_bf16_attn_proj( -- name, -- loaded_weight, -- pending_attn_weights, -- params_dict, -- loaded_params, + if name in ( + "lm_head.weight", + "model.lm_head.weight", + "language_model.lm_head.weight", - ): ++ ): + if self.has_own_lm_head: + for layer_idx in self.model.layers: + head_name = f"model.layers.{layer_idx}.shared_head.head.weight" @@ -3876,181 +657,22 @@ index 90bdb5b642144d2e5a29b9d2e7cb8d8c9277c71c..407ed1f21a72923fd000d564ab7d66f2 + param = params_dict[head_name] + param.weight_loader(param, loaded_weight) + loaded_params.add(head_name) - continue -- -- # Dequantize legacy block-FP8 projections kept in BF16. -- if _try_load_fp8_attn_proj( -- name, -- loaded_weight, -- pending_attn_weights, -- params_dict, -- loaded_params, -- kv_a_pad_size, -- ): -- continue -- -- for param_name, weight_name, shard_id in stacked_params_mapping: -- if weight_name not in name: -- continue -- if ("mlp.experts." in name) and name not in params_dict: -- continue -- name_mapped = name.replace(weight_name, param_name) -- if ( -- param_name == "fused_qkv_a_proj" -- ) and name_mapped not in params_dict: -- continue -- else: -- name = name_mapped -- if name.endswith(".bias") and name not in params_dict: -- continue -- param = params_dict[name] -- weight_loader = param.weight_loader -- weight_loader(param, loaded_weight, shard_id) -- break -- else: -- is_expert_weight = False -- for mapping in expert_params_mapping: -- param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment] -- if weight_name not in name: -- continue -- is_expert_weight = True -- name_mapped = name.replace(weight_name, param_name) -- param = params_dict[name_mapped] -- weight_loader = typing.cast( -- Callable[..., bool], param.weight_loader -- ) -- success = weight_loader( -- param, -- loaded_weight, -- name_mapped, -- shard_id=shard_id, -- expert_id=expert_id, -- return_success=True, -- ) -- if success: -- name = name_mapped -- break -- else: -- if is_expert_weight: -- continue -- if name.endswith(".bias") and name not in params_dict: -- continue -- name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment] -- if name is None: -- continue -- if ( -- spec_layer != self.model.mtp_start_layer_idx -- and ".layers" not in name -- ): -- continue -- param = params_dict[name] -- weight_loader = getattr( -- param, "weight_loader", default_weight_loader -- ) -- weight_loader(param, loaded_weight) -- loaded_params.add(name) -- -+ spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) -+ if spec_layer is None: -+ continue -+ name = self._rewrite_spec_layer_name(spec_layer, name) -+ -+ if _try_load_mxfp8_bf16_attn_proj( -+ name, -+ loaded_weight, -+ pending_attn_weights, -+ params_dict, -+ loaded_params, -+ ): + continue -+ -+ # Dequantize legacy block-FP8 projections kept in BF16. -+ if _try_load_fp8_attn_proj( -+ name, -+ loaded_weight, -+ pending_attn_weights, -+ params_dict, -+ loaded_params, -+ kv_a_pad_size, -+ ): -+ continue -+ -+ for param_name, weight_name, shard_id in stacked_params_mapping: -+ if weight_name not in name: -+ continue -+ if ("mlp.experts." in name) and name not in params_dict: -+ continue -+ name_mapped = name.replace(weight_name, param_name) -+ if ( -+ param_name == "fused_qkv_a_proj" -+ ) and name_mapped not in params_dict: -+ continue -+ else: -+ name = name_mapped -+ if name.endswith(".bias") and name not in params_dict: -+ continue -+ param = params_dict[name] -+ weight_loader = param.weight_loader -+ weight_loader(param, loaded_weight, shard_id) -+ break -+ else: -+ is_expert_weight = False -+ for mapping in expert_params_mapping: -+ param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment] -+ if weight_name not in name: -+ continue -+ is_expert_weight = True -+ name_mapped = name.replace(weight_name, param_name) -+ param = params_dict[name_mapped] -+ weight_loader = typing.cast( -+ Callable[..., bool], param.weight_loader -+ ) -+ success = weight_loader( -+ param, -+ loaded_weight, -+ name_mapped, -+ shard_id=shard_id, -+ expert_id=expert_id, -+ return_success=True, -+ ) -+ if success: -+ name = name_mapped -+ break -+ else: -+ if is_expert_weight: -+ continue -+ if name.endswith(".bias") and name not in params_dict: -+ continue -+ name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment] -+ if name is None: -+ continue -+ if ( -+ spec_layer != self.model.mtp_start_layer_idx -+ and ".layers" not in name -+ ): -+ continue -+ param = params_dict[name] -+ weight_loader = getattr( -+ param, "weight_loader", default_weight_loader -+ ) -+ weight_loader(param, loaded_weight) -+ loaded_params.add(name) -+ + spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) + if spec_layer is None: + continue +@@ -460,6 +491,8 @@ + loaded_layers: set[int] = set() for param_name in loaded_params: + if param_name.endswith(".shared_head.head.weight"): + continue spec_layer = get_spec_layer_idx_from_weight_name(self.config, param_name) -- if spec_layer is not None: -- loaded_layers.add(spec_layer) -- for layer_idx in range( -- self.model.mtp_start_layer_idx, -- self.model.mtp_start_layer_idx + self.model.num_mtp_layers, -+ if spec_layer is not None: -+ loaded_layers.add(spec_layer) -+ for layer_idx in range( -+ self.model.mtp_start_layer_idx, -+ self.model.mtp_start_layer_idx + self.model.num_mtp_layers, + if spec_layer is not None: + loaded_layers.add(spec_layer) +@@ -467,6 +500,13 @@ + self.model.mtp_start_layer_idx, + self.model.mtp_start_layer_idx + self.model.num_mtp_layers, ): + if self.has_own_lm_head: + head_name = f"model.layers.{layer_idx}.shared_head.head.weight" @@ -4060,21 +682,11 @@ index 90bdb5b642144d2e5a29b9d2e7cb8d8c9277c71c..407ed1f21a72923fd000d564ab7d66f2 + "proposal head or target lm_head.weight in the checkpoint." + ) if layer_idx not in loaded_layers: -- raise ValueError( -- f"MTP speculative decoding layer {layer_idx} weights " -- f"missing from checkpoint." -- ) -- return loaded_params -+ raise ValueError( -+ f"MTP speculative decoding layer {layer_idx} weights " -+ f"missing from checkpoint." -+ ) -+ return loaded_params -diff --git a/vllm/utils/b12x.py b/vllm/utils/b12x.py -index aef960a475a5a8cee7f7ca27d70735af55265ba6..e6c794a84a4f48ed277de3e04203e71bae8827d4 100644 + raise ValueError( + f"MTP speculative decoding layer {layer_idx} weights " --- a/vllm/utils/b12x.py +++ b/vllm/utils/b12x.py -@@ -7,10 +7,12 @@ import importlib.util +@@ -7,9 +7,11 @@ from collections.abc import Callable, Hashable, Iterable from dataclasses import dataclass, fields, is_dataclass from types import ModuleType @@ -4082,27 +694,24 @@ index aef960a475a5a8cee7f7ca27d70735af55265ba6..e6c794a84a4f48ed277de3e04203e71b +from typing import Any, Literal import torch - -+import vllm.envs as envs + ++import vllm.envs as envs + @dataclass(frozen=True) - class B12xWarmupUnit: -@@ -19,6 +21,12 @@ class B12xWarmupUnit: +@@ -17,6 +19,12 @@ + name: str + key: Hashable compile: Callable[[], None] - - ++ ++ +def get_b12x_dense_activation_mode(recipe: Literal["nvfp4", "mxfp8"]) -> str: + """Resolve the dense precision override once when loading a layer.""" + override = getattr(envs, f"VLLM_B12X_{recipe.upper()}_ACTIVATION_MODE") + return override if override is not None else envs.VLLM_B12X_DENSE_ACTIVATION_MODE -+ -+ - _HAS_B12X = importlib.util.find_spec("b12x") is not None -diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py -index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae9275d456 100644 + _HAS_B12X = importlib.util.find_spec("b12x") is not None --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -2,11 +2,13 @@ @@ -4119,7 +728,7 @@ index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae from vllm.config import VllmConfig from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import ( -@@ -101,6 +103,7 @@ class GDNAttentionMetadata: +@@ -101,6 +103,7 @@ seq_lens: torch.Tensor | None = None prefill_checkpoint: GDNPrefillCheckpointMetadata | None = None @@ -4127,7 +736,7 @@ index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]): -@@ -108,7 +111,9 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] +@@ -108,7 +111,9 @@ _cudagraph_support = AttentionCGSupport.UNIFORM_BATCH supports_update_block_table: bool = True @@ -4137,10 +746,11 @@ index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae reorder_batch_threshold: int = 1 -@@ -190,6 +195,84 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] +@@ -189,6 +194,84 @@ + (self.decode_cudagraph_max_bs,), dtype=torch.int32, device=device, - ) ++ ) + self._decode_state_indices_source: torch.Tensor | None = None + self._decode_state_indices_view: torch.Tensor | None = None + self._reuse_spec_decode_inputs = envs.VLLM_GDN_SPEC_DECODE_METADATA_FASTPATH @@ -4218,11 +828,10 @@ index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae + num_reqs=num_reqs, + seq_lens=m.seq_lens, + is_uniform_spec_decode=True, -+ ) + ) def _get_state_indices( - self, -@@ -209,6 +292,13 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] +@@ -209,6 +292,13 @@ self.vllm_config.cache_config.mamba_cache_mode, ) @@ -4236,7 +845,7 @@ index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae def _build_chunk_metadata( self, prefill_query_start_loc: torch.Tensor, -@@ -260,6 +350,11 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] +@@ -260,6 +350,11 @@ fast_build: bool = False, ) -> GDNAttentionMetadata: m = common_attn_metadata @@ -4248,7 +857,7 @@ index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae query_start_loc = m.query_start_loc query_start_loc_cpu = m.query_start_loc_cpu -@@ -599,6 +694,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] +@@ -599,6 +694,7 @@ and num_prefills == 0 and num_spec_decodes == 0 and num_decodes <= self.decode_cudagraph_max_bs @@ -4256,7 +865,7 @@ index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae ): self.non_spec_state_indices_tensor[:num_decodes].copy_( non_spec_state_indices_tensor, non_blocking=True -@@ -657,6 +753,41 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] +@@ -657,6 +753,41 @@ assert metadata.num_reqs > 0 assert metadata.seq_lens is not None @@ -4298,7 +907,7 @@ index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae state_indices = self._get_state_indices( blk_table, metadata.seq_lens, -@@ -753,6 +884,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] +@@ -753,6 +884,7 @@ and metadata.num_prefills == 0 and metadata.num_spec_decodes == 0 and metadata.num_decodes <= self.decode_cudagraph_max_bs @@ -4306,11 +915,9 @@ index c6ccbd353ad762af98a3a476f1760a2abcdb5066..d28a3d9e5f6bbff8e7f79ae7196f99ae ): self.non_spec_state_indices_tensor[: metadata.num_decodes].copy_( non_spec_state_indices[: metadata.num_decodes], non_blocking=True -diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py -index e225c78ffe1989bffaf33a4b3a4da3feab19bf75..cbecda1e6c33b2c0a7298eb01ed38d2bb11b1ed7 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py -@@ -293,43 +293,11 @@ def build_attn_metadata( +@@ -293,43 +293,11 @@ attn_metadata: dict[str, Any] = {} cached_attn_metadata: dict[tuple[KVCacheSpec, type], Any] = {} num_kv_cache_groups = len(kv_cache_config.kv_cache_groups) @@ -4357,7 +964,7 @@ index e225c78ffe1989bffaf33a4b3a4da3feab19bf75..cbecda1e6c33b2c0a7298eb01ed38d2b for attn_group in attn_groups[i]: attn_metadata_builder = attn_group.get_metadata_builder(0) -@@ -337,35 +305,74 @@ def build_attn_metadata( +@@ -337,35 +305,74 @@ if isinstance(kv_cache_spec, UniformTypeKVCacheSpecs): kv_cache_spec = kv_cache_spec.kv_cache_specs[attn_group.layer_names[0]] cache_key = (kv_cache_spec, type(attn_metadata_builder)) @@ -4383,13 +990,7 @@ index e225c78ffe1989bffaf33a4b3a4da3feab19bf75..cbecda1e6c33b2c0a7298eb01ed38d2b - model_specific_attn_metadata.get_extra_attn_kwargs( - attn_metadata_builder, - num_reqs, -+ if common_attn_metadata is None: -+ # Per-group causal for hybrid drafters (mixed SWA/full attention). -+ group_causal = ( -+ causal -+ if isinstance(causal, (bool, torch.Tensor)) -+ else causal.get(i, True) - ) +- ) - if model_specific_attn_metadata is not None - else {} - ) @@ -4400,6 +1001,13 @@ index e225c78ffe1989bffaf33a4b3a4da3feab19bf75..cbecda1e6c33b2c0a7298eb01ed38d2b - ) - if attn_metadata_builder.supports_update_block_table: - cached_attn_metadata[cache_key] = metadata ++ if common_attn_metadata is None: ++ # Per-group causal for hybrid drafters (mixed SWA/full attention). ++ group_causal = ( ++ causal ++ if isinstance(causal, (bool, torch.Tensor)) ++ else causal.get(i, True) ++ ) + + common_attn_metadata_extra_kwargs = ( + model_specific_attn_metadata.get_extra_common_attn_kwargs( @@ -4455,11 +1063,9 @@ index e225c78ffe1989bffaf33a4b3a4da3feab19bf75..cbecda1e6c33b2c0a7298eb01ed38d2b for layer_name in attn_group.layer_names: attn_metadata[layer_name] = metadata return attn_metadata -diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py -index 7f4219288bb402703200e0edfbe55d43ff7b1dd0..532ce32e16f740f069731720ba0840312a96137d 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py -@@ -7,6 +7,7 @@ import numpy as np +@@ -7,6 +7,7 @@ import torch import torch.nn as nn @@ -4467,7 +1073,7 @@ index 7f4219288bb402703200e0edfbe55d43ff7b1dd0..532ce32e16f740f069731720ba084031 from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode from vllm.triton_utils import tl, triton -@@ -84,6 +85,11 @@ class MambaHybridModelState(DefaultModelState): +@@ -84,6 +85,11 @@ self.num_accepted_tokens_gpu = torch.ones( self.max_num_reqs, dtype=torch.int32, device=self.device ) @@ -4479,7 +1085,7 @@ index 7f4219288bb402703200e0edfbe55d43ff7b1dd0..532ce32e16f740f069731720ba084031 # Pre-copy "align" prefix-cache state (V2). The migration of each # request's mamba state across block boundaries runs as a fused GPU # kernel reusing the postprocess copy machinery, so the per-step src -@@ -106,6 +112,9 @@ class MambaHybridModelState(DefaultModelState): +@@ -106,6 +112,9 @@ self._mamba_group_ids: list[int] = [] self._mamba_spec: MambaSpec | None = None self._mamba_copy_funcs_by_type: MambaStateCopyFuncsByType | None = None @@ -4489,7 +1095,7 @@ index 7f4219288bb402703200e0edfbe55d43ff7b1dd0..532ce32e16f740f069731720ba084031 def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: super().add_request(req_index, new_req_data) -@@ -122,6 +131,9 @@ class MambaHybridModelState(DefaultModelState): +@@ -122,6 +131,9 @@ if self._align_mode: self._mamba_ctx = None self._mamba_copy_funcs_by_type = None @@ -4499,10 +1105,11 @@ index 7f4219288bb402703200e0edfbe55d43ff7b1dd0..532ce32e16f740f069731720ba084031 if self.recoverssm is not None: self.recoverssm.reset() -@@ -171,6 +183,40 @@ class MambaHybridModelState(DefaultModelState): +@@ -170,6 +182,40 @@ + [block_tables[gid] for gid in mamba_group_ids], ) return ctx - ++ + def _prepare_aligned_state_indices( + self, + seq_lens: torch.Tensor, @@ -4536,11 +1143,10 @@ index 7f4219288bb402703200e0edfbe55d43ff7b1dd0..532ce32e16f740f069731720ba084031 + builder.mamba_aligned_state_indices = group_views[group_idx] + self._aligned_metadata_ctx = ctx + ctx.compute_aligned_state_indices(seq_lens, num_reqs) -+ + def preprocess_state( self, - input_batch: InputBatch, -@@ -277,22 +323,13 @@ class MambaHybridModelState(DefaultModelState): +@@ -277,22 +323,13 @@ num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np) if self._align_mode: @@ -4570,11 +1176,388 @@ index 7f4219288bb402703200e0edfbe55d43ff7b1dd0..532ce32e16f740f069731720ba084031 mamba_attn_metadata = MambaHybridAttnMetadata( is_prefilling=is_prefilling, -diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py -index 4857795fab43ff2daedec18d6acec301a376f825..3e92514f68a6a19a3bb1f91a631f429450226414 100644 +--- a/vllm/v1/worker/gpu/sample/gumbel.py ++++ b/vllm/v1/worker/gpu/sample/gumbel.py +@@ -12,6 +12,12 @@ + # available — on the CPU worker path `tl` is a placeholder whose `constexpr` + # attribute is `None`, and `tl.constexpr(...)` would crash at import time. + _TL_RAND_MIN = tl.constexpr(4.6566127342e-10) if HAS_TRITON else 4.6566127342e-10 ++ ++# Offset salt keeping the draft's Gumbel noise disjoint from the target's. ++# Verification is a probability-ratio test, not a Gumbel coupling, so a proposal ++# and the residual it is resampled from must not share a noise vector. ++# Positions are int64 and never approach 2**30, so the streams cannot collide. ++_DRAFT_NOISE_SALT = tl.constexpr(1 << 30) if HAS_TRITON else (1 << 30) + + + @triton.jit +@@ -89,6 +95,7 @@ + seed, + pos, + temp, ++ IS_DRAFTING: tl.constexpr, + USE_FP64: tl.constexpr, + APPLY_TEMPERATURE: tl.constexpr = True, + ): +@@ -108,6 +115,8 @@ + if USE_FP64: + logits = logits.to(tl.float64) + if temp != 0.0: ++ if IS_DRAFTING: ++ pos = pos + _DRAFT_NOISE_SALT + gumbel_seed = tl.randint(seed, pos) + if USE_FP64: + u = tl_rand64(gumbel_seed, keys, includes_zero=False) +@@ -137,6 +146,7 @@ + logits_cache_stride_1, + logits_cache_col_ptr, + vocab_size, ++ IS_DRAFTING: tl.constexpr, + APPLY_TEMPERATURE: tl.constexpr, + USE_FP64: tl.constexpr, + PER_TOKEN_COL: tl.constexpr = False, +@@ -173,6 +183,7 @@ + seed, + pos, + temp, ++ IS_DRAFTING=IS_DRAFTING, + USE_FP64=USE_FP64, + APPLY_TEMPERATURE=APPLY_TEMPERATURE, + ) +@@ -197,6 +208,7 @@ + temp_ptr, + vocab_size, + BLOCK_SIZE: tl.constexpr, ++ IS_DRAFTING: tl.constexpr, + APPLY_TEMPERATURE: tl.constexpr, + USE_FP64: tl.constexpr, + PER_TOKEN_COL: tl.constexpr, +@@ -226,6 +238,7 @@ + logits_cache_stride_1, + logits_cache_col_ptr, + vocab_size, ++ IS_DRAFTING=IS_DRAFTING, + APPLY_TEMPERATURE=APPLY_TEMPERATURE, + USE_FP64=USE_FP64, + PER_TOKEN_COL=PER_TOKEN_COL, +@@ -242,6 +255,7 @@ + seed: torch.Tensor, # [max_num_reqs] + pos: torch.Tensor, # [num_tokens] + apply_temperature: bool, ++ is_drafting: bool, + logits_cache: torch.Tensor | None = None, # [max_num_reqs, num_cols, vocab_size] + logits_cache_col: torch.Tensor | None = None, # scalar or [num_tokens] + use_fp64: bool = False, +@@ -281,6 +295,7 @@ + temperature, + vocab_size, + BLOCK_SIZE=BLOCK_SIZE, ++ IS_DRAFTING=is_drafting, + APPLY_TEMPERATURE=apply_temperature, + USE_FP64=use_fp64, + PER_TOKEN_COL=per_token_col, +--- a/vllm/v1/worker/gpu/sample/sampler.py ++++ b/vllm/v1/worker/gpu/sample/sampler.py +@@ -317,6 +317,7 @@ + self.sampling_states.seeds.gpu, + pos, + apply_temperature=False, ++ is_drafting=False, + use_fp64=self.use_fp64_gumbel, + ) + return sampled, processed_logits +--- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py ++++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +@@ -46,6 +46,9 @@ + ) + self.current_draft_step = torch.tensor(0, dtype=torch.int64, device=device) + self.last_token_indices = torch.zeros( ++ self.max_num_reqs, dtype=torch.int64, device=device ++ ) ++ self.sample_src_positions = torch.zeros( + self.max_num_reqs, dtype=torch.int64, device=device + ) + +@@ -370,6 +373,7 @@ + input_batch.seq_lens, + num_rejected, + self.input_buffers, ++ self.sample_src_positions, + self.max_model_len, + self.max_num_reqs, + advance_draft_positions=self.advance_draft_positions, +@@ -504,6 +508,8 @@ + ) -> None: + last_token_indices = self.last_token_indices[:num_reqs] + positions = self.input_buffers.positions[last_token_indices] ++ # Hidden state P and token P+1 predict P+2; key sampling by P+1. ++ sample_src_positions = positions + 1 + idx_mapping = self.idx_mapping[:num_reqs] + + last_hidden_states, hidden_states = self._run_model( +@@ -523,7 +529,7 @@ + + self.draft_tokens[:num_reqs, 0] = self.sample_draft( + sample_hidden_states, +- positions, ++ sample_src_positions, + idx_mapping, + self.temperature, + self.seeds, +@@ -543,6 +549,7 @@ + num_reqs, + ) + self.input_buffers.positions[:num_reqs] = positions ++ self.sample_src_positions[:num_reqs] = sample_src_positions + + def _multi_step_decode( + self, +@@ -698,7 +705,6 @@ + self._prepare_eplb_forward(num_reqs) + + idx_mapping = self.idx_mapping[:num_reqs] +- positions = self.input_buffers.positions[:num_reqs] + # Run the draft model forward pass. + last_hidden_states, hidden_states = self._run_model( + num_tokens_padded, +@@ -707,18 +713,12 @@ + num_tokens_across_dp, + cudagraph_runtime_mode, + ) +- last_hidden_states = last_hidden_states[:num_reqs] +- +- sample_positions = positions +- if not self.advance_draft_positions: +- # The forward pass holds positions fixed (Q-only, shared target KV), +- # but Gumbel sampling still needs the absolute draft position. +- sample_positions = positions + self.current_draft_step +- + # Sample the draft tokens. ++ sample_hidden_states = last_hidden_states[:num_reqs] ++ sample_src_positions = self.sample_src_positions[:num_reqs] + draft_tokens = self.sample_draft( +- last_hidden_states, +- sample_positions, ++ sample_hidden_states, ++ sample_src_positions, + idx_mapping, + self.temperature, + self.seeds, +@@ -734,6 +734,7 @@ + self.draft_tokens, + self.hidden_states, + self.input_buffers, ++ self.sample_src_positions, + num_reqs, + self.max_model_len, + self.num_speculative_steps, +@@ -1003,6 +1004,7 @@ + num_rejected_ptr, + input_ids_ptr, + positions_ptr, ++ sample_src_positions_ptr, + mrope_positions_ptr, + mrope_positions_stride, + query_start_loc_ptr, +@@ -1034,6 +1036,10 @@ + draft_token = tl.load(draft_tokens_ptr + req_idx * draft_tokens_stride) + tl.store(input_ids_ptr + req_idx, draft_token) + ++ # Sampling advances even when forward positions clamp at max_model_len. ++ sample_position = tl.load(sample_src_positions_ptr + req_idx) ++ tl.store(sample_src_positions_ptr + req_idx, sample_position + 1) ++ + target_seq_len = tl.load(target_seq_lens_ptr + req_idx) + num_rejected = tl.load(num_rejected_ptr + req_idx) + seq_len = target_seq_len - num_rejected +@@ -1061,6 +1067,7 @@ + target_seq_lens: torch.Tensor, + num_rejected: torch.Tensor, + input_buffers: InputBuffers, ++ sample_src_positions: torch.Tensor, + max_model_len: int, + max_num_reqs: int, + advance_draft_positions: bool = True, +@@ -1080,6 +1087,7 @@ + num_rejected, + input_buffers.input_ids, + input_buffers.positions, ++ sample_src_positions, + mrope_positions, + mrope_positions_stride, + input_buffers.query_start_loc, +@@ -1100,6 +1108,7 @@ + next_input_hidden_states_stride, + input_ids_ptr, + positions_ptr, ++ sample_src_positions_ptr, + mrope_positions_ptr, + mrope_positions_stride, + seq_lens_ptr, +@@ -1128,6 +1137,10 @@ + # This is the final step. Skip updating draft forward inputs. + return + ++ # Sampling advances even when forward positions clamp at max_model_len. ++ sample_position = tl.load(sample_src_positions_ptr + req_idx) ++ tl.store(sample_src_positions_ptr + req_idx, sample_position + 1) ++ + # Write the sampled draft token into the input ids tensor for the next + # forward pass. + tl.store(input_ids_ptr + req_idx, draft_token) +@@ -1177,6 +1190,7 @@ + output_draft_tokens: torch.Tensor, + next_input_hidden_states: torch.Tensor, + input_buffers: InputBuffers, ++ sample_src_positions: torch.Tensor, + num_reqs: int, + max_model_len: int, + num_speculative_steps: int, +@@ -1197,6 +1211,7 @@ + next_input_hidden_states.stride(0), + input_buffers.input_ids, + input_buffers.positions, ++ sample_src_positions, + mrope_positions, + mrope_positions_stride, + input_buffers.seq_lens, +--- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py ++++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +@@ -288,11 +288,11 @@ + ) + num_sample = num_reqs * self.num_speculative_steps + sample_hidden_states = last_hidden_states[self.sample_indices[:num_sample]] +- # sample_pos is the predicted token's position Q; verification keys +- # Gumbel by the predecessor (Q-1). sample_draft adds +1, so pass Q-2. ++ # sample_pos is the predicted token's position P. Sampling keys a draw ++ # by the position before the sampled token, P-1. + draft_tokens = self.sample_draft( + sample_hidden_states, +- self.sample_pos[:num_sample] - 2, ++ self.sample_pos[:num_sample] - 1, + self.sample_idx_mapping[:num_sample], + self.temperature, + self.seeds, +--- a/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py ++++ b/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py +@@ -51,15 +51,17 @@ + other=0, + ) + +- # Candidate ids key the noise, matching the target's own sampling. +- position = tl.load(sample_pos_ptr + flat) - 1 ++ # sample_pos is the predicted token's position P. Sampling keys a draw ++ # by the position before the sampled token, P-1. ++ sample_pos = tl.load(sample_pos_ptr + flat) - 1 + _, index = gumbel_noised_argmax( + scores, + candidates, + mask & valid, + seed, +- position, ++ sample_pos, + temperature if SAMPLE_PROBABILISTIC else 0.0, ++ IS_DRAFTING=True, + USE_FP64=USE_FP64, + ) + +--- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py ++++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +@@ -135,8 +135,8 @@ + buf.index_copy_(1, self._d2t_scatter_index, logits.to(buf.dtype)) + logits = buf + +- # sample_pos is the predicted token's position Q; the target verifies +- # it with the predecessor's Gumbel key (Q-1). Pass Q-1. ++ # sample_pos is the predicted token's position P. Sampling keys a draw ++ # by the position before the sampled token, P-1. + return gumbel_sample( + logits, + idx_map, +@@ -144,6 +144,7 @@ + self.seeds, + sample_pos - 1, + apply_temperature=True, ++ is_drafting=True, + logits_cache=self.draft_logits, + logits_cache_col=self._step_cols[step], + use_fp64=self.use_fp64_gumbel, +--- a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py ++++ b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py +@@ -377,7 +377,11 @@ + cudagraph_runtime_mode: CUDAGraphMode = CUDAGraphMode.NONE, + ) -> None: + last_token_indices = self.last_token_indices[:num_reqs] +- sample_positions = self.input_buffers.positions[last_token_indices] ++ positions = self.input_buffers.positions[last_token_indices] ++ # The output hidden state at position P (= positions) and the token id ++ # at P+1 are used to draft the token at P+2. Sampling keys a draw by the ++ # position before the sampled token, so the net adjustment is +1. ++ sample_src_positions = positions + 1 + idx_mapping = self.idx_mapping[:num_reqs] + + # Cache the trailing token's ids, hidden states (and embeddings for +@@ -415,7 +419,7 @@ + sample_hidden_states = last_hidden_states[last_token_indices] + draft_tokens = self.sample_draft( + sample_hidden_states, +- sample_positions, ++ sample_src_positions, + idx_mapping, + self.temperature, + self.seeds, +@@ -446,7 +450,8 @@ + idx_mapping, + num_reqs, + ) +- sample_positions += 1 ++ # Advance the draft sampling key. ++ sample_src_positions += 1 + + + @triton.jit +--- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py ++++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py +@@ -837,6 +837,7 @@ + 0, # logits_cache_stride_1 + None, # logits_cache_col_ptr + vocab_size, ++ IS_DRAFTING=False, + APPLY_TEMPERATURE=False, + USE_FP64=USE_FP64, + ) +--- a/vllm/v1/worker/gpu/spec_decode/speculator.py ++++ b/vllm/v1/worker/gpu/spec_decode/speculator.py +@@ -391,7 +391,7 @@ + def sample_draft( + self, + hidden_states: torch.Tensor, +- positions: torch.Tensor, ++ sample_src_positions: torch.Tensor, + idx_mapping: torch.Tensor, + temperature: torch.Tensor, + seeds: torch.Tensor, +@@ -400,15 +400,14 @@ + ) -> torch.Tensor: + if draft_logits is not None: + logits = self.model.compute_logits(hidden_states) +- # NOTE(woosuk): We must add 1 to the positions to match the Gumbel noise +- # used for draft and target sampling. + return gumbel_sample( + logits, + idx_mapping, + temperature, + seeds, +- positions + 1, ++ sample_src_positions, + apply_temperature=True, ++ is_drafting=True, + logits_cache=draft_logits, + logits_cache_col=draft_step, + use_fp64=self.use_fp64_gumbel, --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py -@@ -85,8 +85,10 @@ def get_aligned_state_indices_multi_group_kernel( +@@ -85,8 +85,10 @@ mask=( valid_group[:, None, None] & valid_row[None, :, None] diff --git a/runtime/glm53-spark-mtp3-mesh/pins.json b/runtime/glm53-spark-mtp3-mesh/pins.json index 9039b84a..2aec2f66 100644 --- a/runtime/glm53-spark-mtp3-mesh/pins.json +++ b/runtime/glm53-spark-mtp3-mesh/pins.json @@ -4,10 +4,10 @@ "image_pins": "../glm53-flash-jj-r8-gb10/pins.json", "compute": { "source_lock": "compute/source-lock.json", - "source_lock_sha256": "2a444f7c1ad4319f64afbffb16403491a4863934372cf818f18c510d58c0d00e", + "source_lock_sha256": "9a9568bc9f6bc34f4ee2ac5ebe881c3b8a6de289b26aabdc4056ea2852e49fd9", "vllm_base_revision": "e02b174693e13859de61811b5e8cd13d5308e259", - "b12x_revision": "b58f34eaf978277621efced6678e6713fd7122e4", - "b12x_tree": "7637fe5fb4d88882e0d18cdacc68c493f478499d", + "b12x_revision": "ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301", + "b12x_tree": "dcf039e5e754136275835ea997e6b9abbb6b15ae", "cuda_version": "13.3" }, "target": { @@ -17,9 +17,39 @@ "index_sha256": "db30fc7c5a70ccfb3b1c46637bb4ddb04226b95a5dfc451dffccb96a4f0ff544", "checkpoint_identity": "357f6a86160ebd5caff25d9a10d9f29e8547b16c6c73e78751fa69fde11ac4e4" }, - "speculation": {"method": "mtp", "num_speculative_tokens": 3, "attention_backend": "B12X", "draft_tensor_parallel_size": 4, "draft_sample_method": "probabilistic", "rejection_sample_method": "standard"}, - "capture_sizes": [4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64], - "captured_sircl_query_rows": [16, 20, 24, 28, 32], + "speculation": { + "method": "mtp", + "num_speculative_tokens": 3, + "attention_backend": "B12X", + "draft_tensor_parallel_size": 4, + "draft_sample_method": "probabilistic", + "rejection_sample_method": "standard" + }, + "capture_sizes": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 36, + 40, + 44, + 48, + 52, + 56, + 60, + 64 + ], + "captured_sircl_query_rows": [ + 16, + 20, + 24, + 28, + 32 + ], "canonical_bundle_manifest_sha256": "69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500", "measured_bundle_manifest_sha256": "701bdc42069a97492981b8f34e006ebfa9e68c2160472cba631b56965efae226", "bundle_difference": "The serving measurements use the recorded measured bundle. The canonical bundle additionally preserves capture-ID stream guards and orders shared staging/output copies across streams; routing and collective algorithms are unchanged.", diff --git a/runtime/glm53-spark-mtp3-mesh/test_image.py b/runtime/glm53-spark-mtp3-mesh/test_image.py index 0d2ba490..5bef8af6 100644 --- a/runtime/glm53-spark-mtp3-mesh/test_image.py +++ b/runtime/glm53-spark-mtp3-mesh/test_image.py @@ -363,7 +363,7 @@ def test_profile_pins_exact_compute_source_and_quantization_environment(): "VLLM_LM_HEAD_A16": "1", "VLLM_MXFP8_LM_HEAD": "0", } - assert len(lock["vllm"]["files"]) == 14 + assert len(lock["vllm"]["files"]) == 24 def test_schema_accepts_research_only_status(): From 001c48cec568f37e89deac033afd96a2aa3cca02 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:00:24 -0500 Subject: [PATCH 07/16] Bind GLM mesh profile to the validated compute and stream-safety image --- README.md | 4 +- docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md | 65 +- ...k-mtp3-compute-stream-safety-20260906.json | 4103 +++++++++++++++++ ...ark-mtp3-compute-stream-safety-20260906.md | 63 + .../glm53-spark-mtp3-managed-mesh-tp4.json | 47 +- runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md | 17 +- runtime/glm53-spark-mtp3-mesh/README.md | 35 +- .../compute-image-equivalence.json | 11 +- .../glm53-spark-mtp3-mesh/compute/README.md | 6 +- .../compute/source-lock.json | 5 +- .../glm53-spark-mtp3-mesh/image-receipt.json | 29 +- .../glm53-spark-mtp3-mesh/managed_install.py | 1 + runtime/glm53-spark-mtp3-mesh/pins.json | 6 +- .../glm53-spark-mtp3-mesh/public-image.json | 14 +- .../qualification/run_native.py | 6 +- .../qualification/stream_roce.py | 81 + 16 files changed, 4386 insertions(+), 107 deletions(-) create mode 100644 performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.json create mode 100644 performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md create mode 100644 runtime/glm53-spark-mtp3-mesh/qualification/stream_roce.py diff --git a/README.md b/README.md index 9c3b25e1..5fc8f721 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,9 @@ four-row increments through 64 rows. The image composition uses CUDA 13.3, the native-MTP3 metadata port derived from Local Inference Lab vLLM revision `3512b066`, and the complete B12X tree at -`b58f34ea` with vLLM integration based on `a8c796f3`. The complete B12X update +`ef308bac` with source-checked top-k selector files and vLLM integration based +on `a8c796f3`. Deferred weights own their storage, and draft/rejection sampling +uses independent randomness. The complete B12X update also includes MoE and dense-precision work, so comparisons across different compute configurations cannot attribute a gain to dense kernels alone. The [head-specific comparison](performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md) diff --git a/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md b/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md index 92c7d3da..8df5385c 100644 --- a/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md +++ b/docs/GLM53_SPARK_MTP3_MESH_QUICKSTART.md @@ -1,21 +1,11 @@ # Run GLM-5.3 Flash Spark with native MTP3 and hardware-forwarded mesh -Status: **research-only**. The profile's composition, managed host service, -and CPU checks are **implemented**. The -[managed functional record](../performance/records/glm53-flash/spark-mtp3-managed-mesh-functional-20260905.md) -qualifies bounded installer, policy-scoped fault/recovery, post-recovery -readiness, and one persistent-cache recall case for the image digest recorded -in that report. -Broader cache/failure coverage and unattended serving remain unqualified. - -The [public application-install record](../performance/records/glm53-flash/spark-mtp3-public-application-install-20260905.md) -covers fresh public checkouts, extracted image artifacts, empty application -caches, installation, native correctness, and model-restart cache restoration -on four prepared hosts. It does not qualify a factory-reset OS/network setup. -These functional records qualify only their recorded image digests. Restart, -cache restoration, and failure containment require validation for the image -pinned in `runtime/glm53-spark-mtp3-mesh/public-image.json`; proposal-head -throughput measurements do not establish those properties. +Status: **research-only** profile with **implemented** source packaging and +managed host services. The published image passed the native, GPU stream, +serving, idle rank-loss, restart, and persistent-recall checks in its +[exact-image validation record](../performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md). +That record defines the qualified conditions; in-flight collective failure +containment and unattended availability are not established. **Starting with four stock Sparks and no image?** Follow [the managed-mesh prerequisite section](PREREQUISITES.md#four-spark-managed-hardware-forwarded-mesh) @@ -36,8 +26,9 @@ DFlash model is used. The [profile contract](../runtime/glm53-spark-mtp3-mesh/README.md) and [pins](../runtime/glm53-spark-mtp3-mesh/pins.json) are the canonical inputs. The packaged RoCEnante runtime orders shared staging buffers across streams -and preserves its one-stream-per-CUDA-capture guard. These fixes have CPU -regression coverage; four-rank GPU fault and stream tests remain required. +and preserves its one-stream-per-CUDA-capture guard. CPU regressions and +four-rank GPU tests cover alternating streams, misaligned buffers, +changed-input graph replay, and second-stream capture rejection. The [proposal-head throughput record](../performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md) reports observations, not a general performance guarantee. @@ -375,16 +366,15 @@ separate from SIRCL's two 64 MiB transport arenas. Native MTP's cache draft identity is the target checkpoint. The profile uses the dedicated namespace -`glm53-spark-df116c4f-mtp3-nvfp4-a16-b58f34ea-mesh4204fabc-tail-cow-v2`; +`glm53-spark-df116c4f-mtp3-nvfp4-a16-c139f3670-mesh69313e19-tail-cow-v2`; shared-BF16-head and external-DFlash entries must not be renamed into it. The `draft_policy=separate` field describes cache registration layout, not an external draft model. The -[managed functional record](../performance/records/glm53-flash/spark-mtp3-managed-mesh-functional-20260905.md) -includes an uncached publication and stopped-container restoration for its -recorded image and namespace. Restoration under the NVFP4-proposal-head -namespace named above is research-only until a stopped-container restore test -passes for that configuration. The linked record covers one recall prompt, -not all context lengths or concurrent cache workloads. +[exact-image validation record](../performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md) +includes publication and restoration of a 26,624-token prefix after all model +containers stopped and restarted under this namespace. The answer, external-hit +counters, and all four restore logs agree. This qualifies the recorded recall +case, not every context length or concurrent cache workload. ## Obtain the image and target @@ -400,13 +390,13 @@ The content and registry receipts identify the same public image. The [compute-image equivalence record](../runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json) verifies all 4,891 vLLM, 385 B12X, and 150 SparkCache package files plus the selected environment against tested private image -`sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74`. -That proves build/content equivalence, not a fresh serving, restart, or -persistent-cache qualification for the public image ID. +`sha256:3b4768e5ba31cadcc882dffa06d7b667af44abdf157d5c11b7ac7fe962e80c43`. +The mounted transport is checked separately. The published config-image ID +below is also the exact image used for the linked runtime validation. ```bash -mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:b65d427f9be49c97d57e404ad1a6118769c1119df876a8944c1f186a6b380c5d' -mtp_image_id='sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766' +mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:67dc0ae453baaae6831ccec1d259b4ef8b236a8b0dc9f747d901b95c66ec1987' +mtp_image_id='sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f' docker pull "${mtp_image}" test "$(docker image inspect "${mtp_image}" --format '{{.Id}}')" = "${mtp_image_id}" @@ -657,6 +647,21 @@ mesh service. ### Model output and persistent-cache restoration +For the selected stream-safety cases, use the same stopped-model test window +and rendered image receipt as the native check: + +```bash +python3 runtime/glm53-spark-mtp3-mesh/qualification/run_native.py \ + --launch /srv/sparkring/mtp3-mesh-launch \ + --image-receipt /srv/sparkring/verified-image-receipt.json \ + --mode streams --rows 4 64 --port 30140 \ + --output /path/to/private-receipts/stream-checks --execute-authorized +``` + +This checks alternating caller streams with misaligned buffers, changed-input +graph replay, and rejection of a second stream in one CUDA capture. It does +not inject in-flight link or GPU failures. + Start the four-rank model through `managed_cluster.py start-model` and wait for completed speculation warmup. Set the endpoint to the rank-zero management address: diff --git a/performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.json b/performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.json new file mode 100644 index 00000000..a2fab68d --- /dev/null +++ b/performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.json @@ -0,0 +1,4103 @@ +{ + "schema": "sparkring-glm53-compute-stream-validation/v1", + "status": "qualified", + "scope": "Exact-image native all-reduce, selected GPU stream invariants, bounded serving matrix, idle rank-loss shutdown, restart and one persistent-prefix recall.", + "image": "sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f", + "registry_manifest": "sha256:67dc0ae453baaae6831ccec1d259b4ef8b236a8b0dc9f747d901b95c66ec1987", + "compute_source_lock_sha256": "139f36701e0e47f45bf99fba2cc2fa59b417f2ee801dad3a064455d5b464a459", + "transport_manifest_sha256": "69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500", + "startup_memory_gate": "Host source from draft PR222; no watchdog or memory thresholds relaxed.", + "host_source_manifest": { + "composition": "PR219 compute/profile contracts and PR222 memory-gated host startup", + "files": { + "runtime/glm53-spark-mtp3-mesh/managed_memory.py": "a338586470ebcae67df13bb30d4f6707d2b10da0f67eecaf6c3b5c1fcc22a367", + "runtime/glm53-spark-mtp3-mesh/managed_service.py": "e75dcd5394951a5801f6c5f742d1aef69f91d9fbfbd324be012667b893c4ed08", + "runtime/glm53-spark-mtp3-mesh/managed_network.py": "f2c9538e356537f6e9c63fcbd6ee086d4641491b034b07b8b86860be7c678877", + "runtime/glm53-spark-mtp3-mesh/managed_units.py": "4a8f47de02e2acba28a4a6a0847f95ed9877ee8cf9ad15e040bd82ef7540e786", + "runtime/glm53-spark-mtp3-mesh/managed_cluster.py": "c772836f208f24e8bedb5c6b57c8818146dade2a46b88bc18f974fd1f16b023f", + "runtime/glm53-spark-mtp3-mesh/managed_install.py": "cbb9f89f2ff3d2b2d0957fc50522d48c5813c79815c2b9acc16a82ae3b4b57b0", + "runtime/glm53-spark-mtp3-mesh/profile.py": "9e5e2b8e8bb1a0f74982f816723ebbfed361b2d94c494a50983057167bf367ff", + "runtime/glm53-spark-mtp3-mesh/inspect_fabric.py": "def378fea0534c4302c14b0b22be5d40f7bfe456880748e3ee6c7b7237203cc6", + "runtime/glm53-spark-mtp3-mesh/pins.json": "05fddc68d2e74e00ed29126c09258766aff8a852b67681836023c097234d728b", + "runtime/glm53-flash-jj-r8-gb10/pins.json": "dc49370d911ecc32f1c7ea7656f41837334a8e5ed800db263174781bee2b2313", + "runtime/glm53-flash-jj-r8-gb10/warmup_dflash.py": "f41c38eef41d15d63dcfc49cd6643357ca1a3ae18200ddbe4f8692d0b767ee79", + "runtime/glm53-flash-jj-r8-gb10/launch-rank.sh": "8a4511eb80e3d12d9daccfc9c1d2e6305c67926e24017f4e90528cc3be6c3ba7", + "runtime/glm53-flash-jj-r8-gb10/runtime.env.example": "707e0568d0410e8911efbb15f2d4e97e779ef1ac81712de70265f4e7769cc911", + "runtime/glm53-flash-jj-r8-gb10/sircl-fused.env.example": "f47619ad2aaece7425e1958ddf19fa9b6508fbe0964fb66fe88aee0f864c3038", + "spark_transport/experiments/cx7_hairpin_diagonal/__init__.py": "67fd1b48093286b47f2cf32c7fcd69d3e6d29bf6484abc7216a82da0e10e7b41", + "spark_transport/experiments/cx7_hairpin_diagonal/fabric.py": "cbf2f596e744ab50caac66db41e91c1cd3a5625f3cc6ba77ffbd1d667147d544", + "spark_transport/experiments/glm53_rocenante_overlay/build_bundle.py": "ca5f5502b55fbe095bdcdf1283b1f5df85b0f9727873970df727475d5f305a3b", + "runtime/glm53-spark-mtp3-mesh/compute/source-lock.json": "139f36701e0e47f45bf99fba2cc2fa59b417f2ee801dad3a064455d5b464a459" + } + }, + "native": [ + { + "schema": "b12x.rocenante-virtual-diagonal-evidence/v1", + "status": "research-only", + "payload_bytes": 32768, + "warmups": 2, + "samples": 3, + "graph_operations_per_replay": 3, + "ranks": [ + { + "rank": 0, + "eager_samples_us": [ + 444.41598653793335, + 108.99200290441513, + 66.39999896287918 + ], + "graph_samples_us": [ + 240.48000574111938, + 21.5786670645078, + 21.290667355060577 + ], + "eager_median_us": 108.99200290441513, + "graph_median_us": 21.5786670645078, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "48d35fdc0ee602ef417b66e84d5b8161066b7c09efef2aee301b94678645fa55", + "expected_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "output_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "98a9a14f351766d2743a1b9d2967cbe2f5993ef7d56889f9d78bb2974269267d", + "expected_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "output_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "48d35fdc0ee602ef417b66e84d5b8161066b7c09efef2aee301b94678645fa55", + "expected_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "output_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "f71e11bb4c4c1043db1c26bcb9aa8ff4d2523663e9ff05385e17ef534205e60b", + "expected_sha256": "b282b975648deeef7ecb7491a997047c1f9561901f148450b08ee5b80712328a", + "output_sha256": "b282b975648deeef7ecb7491a997047c1f9561901f148450b08ee5b80712328a", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "98a9a14f351766d2743a1b9d2967cbe2f5993ef7d56889f9d78bb2974269267d", + "expected_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "output_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 0, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 0, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "1": [ + 0, + 2 + ], + "2": [ + 0, + 3 + ], + "3": [ + 1, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 464, + "remote_qp_number": 847, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 457, + "remote_qp_number": 713, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 1081344, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 465, + "remote_qp_number": 847, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 1081344, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 714, + "remote_qp_number": 457, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 849, + "remote_qp_number": 464, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 715, + "remote_qp_number": 457, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 540672, + "payload_bytes_path_1": 540672 + } + }, + { + "rank": 1, + "eager_samples_us": [ + 255.71200251579285, + 119.87199634313583, + 55.80800026655197 + ], + "graph_samples_us": [ + 191.4880077044169, + 20.67199970285098, + 22.133332987626392 + ], + "eager_median_us": 119.87199634313583, + "graph_median_us": 22.133332987626392, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "080d97a7d56f4c6201ed6913d36927d48fdee761dbaba585f2eeee5c93f0d0a8", + "expected_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "output_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "3ee6d0365f9d1e5297cff3c44da13ac69e147d518f411bd33cc9f5e40166c94d", + "expected_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "output_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "080d97a7d56f4c6201ed6913d36927d48fdee761dbaba585f2eeee5c93f0d0a8", + "expected_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "output_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "f64ed4bf2210447de8c95ffa74f5e5839d722b49c3eb63b552dd3c948528dec2", + "expected_sha256": "b282b975648deeef7ecb7491a997047c1f9561901f148450b08ee5b80712328a", + "output_sha256": "b282b975648deeef7ecb7491a997047c1f9561901f148450b08ee5b80712328a", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "3ee6d0365f9d1e5297cff3c44da13ac69e147d518f411bd33cc9f5e40166c94d", + "expected_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "output_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 1, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 0, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 1, + 3 + ], + "2": [ + 0, + 2 + ], + "3": [ + 0, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 847, + "remote_qp_number": 464, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 713, + "remote_qp_number": 457, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 465, + "remote_qp_number": 848, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 458, + "remote_qp_number": 714, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 1081344, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 466, + "remote_qp_number": 848, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 1081344, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 715, + "remote_qp_number": 458, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 2, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 540672, + "payload_bytes_path_1": 540672 + } + }, + { + "rank": 2, + "eager_samples_us": [ + 207.20000565052032, + 125.2799928188324, + 49.15200173854828 + ], + "graph_samples_us": [ + 42.87999868392944, + 20.810666183630627, + 21.877333521842957 + ], + "eager_median_us": 125.2799928188324, + "graph_median_us": 21.877333521842957, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "52ed8ae4094366d3b404eda20e086af8c2ea7673c41993998ab566c3c751df7b", + "expected_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "output_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "70e5ef9e0c8aa7e0ba8e4276d397164259c3cfa25b0de39f76b4d3e6a37c8329", + "expected_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "output_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "52ed8ae4094366d3b404eda20e086af8c2ea7673c41993998ab566c3c751df7b", + "expected_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "output_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "6ba9e176e052357b0bc90b26f9d5625780e57bcfcf091ccc891d4265273baf08", + "expected_sha256": "b282b975648deeef7ecb7491a997047c1f9561901f148450b08ee5b80712328a", + "output_sha256": "b282b975648deeef7ecb7491a997047c1f9561901f148450b08ee5b80712328a", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "70e5ef9e0c8aa7e0ba8e4276d397164259c3cfa25b0de39f76b4d3e6a37c8329", + "expected_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "output_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 2, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 0, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 1, + 2 + ], + "1": [ + 1, + 3 + ], + "3": [ + 0, + 2 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 1081344, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 847, + "remote_qp_number": 465, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 1081344, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 457, + "remote_qp_number": 714, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 848, + "remote_qp_number": 465, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 714, + "remote_qp_number": 458, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 466, + "remote_qp_number": 849, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 459, + "remote_qp_number": 715, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 540672, + "payload_bytes_path_1": 540672 + } + }, + { + "rank": 3, + "eager_samples_us": [ + 50.464000552892685, + 138.84800672531128, + 54.1439987719059 + ], + "graph_samples_us": [ + 113.7600044409434, + 22.154666483402252, + 21.205333371957142 + ], + "eager_median_us": 54.1439987719059, + "graph_median_us": 22.154666483402252, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "f2fd2757da7269e75a3e951341d0aca81d50ffeb15cb777beabb60cdb57c23a6", + "expected_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "output_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "6bb75bbeb7bf2338fbfa910d2cc216be460410287c94fa595007b348b0ed5a18", + "expected_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "output_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "f2fd2757da7269e75a3e951341d0aca81d50ffeb15cb777beabb60cdb57c23a6", + "expected_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "output_sha256": "202a06d669dc01f7432eda3e514cf5740ea9652f609c8e0dfec40716b2651522", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "b206433a2115b49319dc85427a12a65c4651bf36545ec41cdb522d6c2516a1bb", + "expected_sha256": "b282b975648deeef7ecb7491a997047c1f9561901f148450b08ee5b80712328a", + "output_sha256": "b282b975648deeef7ecb7491a997047c1f9561901f148450b08ee5b80712328a", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "6bb75bbeb7bf2338fbfa910d2cc216be460410287c94fa595007b348b0ed5a18", + "expected_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "output_sha256": "d6de16041970b40bf4c7653ad60f95ae46976f16550b1d54988463980a19fbc2", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 3, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 0, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 0, + 2 + ], + "1": [ + 1, + 2 + ], + "2": [ + 1, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 464, + "remote_qp_number": 849, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 457, + "remote_qp_number": 715, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 1081344, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 848, + "remote_qp_number": 466, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 1081344, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 458, + "remote_qp_number": 715, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 849, + "remote_qp_number": 466, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 540672, + "physical_hop_payload_bytes": 540672, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 715, + "remote_qp_number": 459, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 540672, + "payload_bytes_path_1": 540672 + } + } + ] + }, + { + "schema": "b12x.rocenante-virtual-diagonal-evidence/v1", + "status": "research-only", + "payload_bytes": 163840, + "warmups": 2, + "samples": 3, + "graph_operations_per_replay": 3, + "ranks": [ + { + "rank": 0, + "eager_samples_us": [ + 1161.5999937057495, + 145.31199634075165, + 79.13599908351898 + ], + "graph_samples_us": [ + 122.1440037091573, + 61.86666587988535, + 77.60000228881836 + ], + "eager_median_us": 145.31199634075165, + "graph_median_us": 77.60000228881836, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "c5353ec97297a7430ac63087c23328c4a6c7870de568a57ec33cb3f5e24b1423", + "expected_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "output_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "12c37dce350901b16b44060cfc088f8347c7e165e1e6b4ee37461ddc2b0ce427", + "expected_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "output_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "c5353ec97297a7430ac63087c23328c4a6c7870de568a57ec33cb3f5e24b1423", + "expected_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "output_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "0400d22ed348280b33c32240f6dad0dbdc4e01736f5dd9daca54126ea27f4888", + "expected_sha256": "a773986c394cf2f8b67267812f58785f84fd7be5ceace0f32ce3d8730abbef1a", + "output_sha256": "a773986c394cf2f8b67267812f58785f84fd7be5ceace0f32ce3d8730abbef1a", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "12c37dce350901b16b44060cfc088f8347c7e165e1e6b4ee37461ddc2b0ce427", + "expected_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "output_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 0, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 0, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "1": [ + 0, + 2 + ], + "2": [ + 0, + 3 + ], + "3": [ + 1, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 467, + "remote_qp_number": 850, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 460, + "remote_qp_number": 716, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 5406720, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 468, + "remote_qp_number": 850, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 5406720, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 717, + "remote_qp_number": 460, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 852, + "remote_qp_number": 467, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 718, + "remote_qp_number": 460, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 2703360, + "payload_bytes_path_1": 2703360 + } + }, + { + "rank": 1, + "eager_samples_us": [ + 255.16799092292786, + 152.0639955997467, + 5669.312000274658 + ], + "graph_samples_us": [ + 86.66666348775227, + 58.84799857934316, + 71.19999825954437 + ], + "eager_median_us": 255.16799092292786, + "graph_median_us": 71.19999825954437, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "6664adc78c736e7e12488891b5cce2f17a973e4afbf5281c941827c0cb5d12f5", + "expected_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "output_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "75fbaf49ad48bcbe0f9410ce15291b79664a7fd94f9c5148ffaf5bf20a15fadf", + "expected_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "output_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "6664adc78c736e7e12488891b5cce2f17a973e4afbf5281c941827c0cb5d12f5", + "expected_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "output_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "4b38daba9747bc84a8bf035e50531c0b182ca834a2a28a00b55317f9c75db659", + "expected_sha256": "a773986c394cf2f8b67267812f58785f84fd7be5ceace0f32ce3d8730abbef1a", + "output_sha256": "a773986c394cf2f8b67267812f58785f84fd7be5ceace0f32ce3d8730abbef1a", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "75fbaf49ad48bcbe0f9410ce15291b79664a7fd94f9c5148ffaf5bf20a15fadf", + "expected_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "output_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 1, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 0, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 1, + 3 + ], + "2": [ + 0, + 2 + ], + "3": [ + 0, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 850, + "remote_qp_number": 467, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 716, + "remote_qp_number": 460, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 468, + "remote_qp_number": 851, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 461, + "remote_qp_number": 717, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 5406720, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 469, + "remote_qp_number": 851, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 5406720, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 718, + "remote_qp_number": 461, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 2, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 2703360, + "payload_bytes_path_1": 2703360 + } + }, + { + "rank": 2, + "eager_samples_us": [ + 707.647979259491, + 115.55200070142746, + 120.4800009727478 + ], + "graph_samples_us": [ + 271.8613346417745, + 59.14666752020518, + 79.83999947706859 + ], + "eager_median_us": 120.4800009727478, + "graph_median_us": 79.83999947706859, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "b83703e6cd9c085f96af128d6f27dcc084689dbd778a4c319f836b871faa29ea", + "expected_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "output_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "3e8d4c9ba4b5404b015873cb17affd09878c8cc461e7c81c759b4ffe6e9f9792", + "expected_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "output_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "b83703e6cd9c085f96af128d6f27dcc084689dbd778a4c319f836b871faa29ea", + "expected_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "output_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "8aed95419a9fe77e6131f40d29c181ff843bedacd4d2a9687f38d211c4c256a1", + "expected_sha256": "a773986c394cf2f8b67267812f58785f84fd7be5ceace0f32ce3d8730abbef1a", + "output_sha256": "a773986c394cf2f8b67267812f58785f84fd7be5ceace0f32ce3d8730abbef1a", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "3e8d4c9ba4b5404b015873cb17affd09878c8cc461e7c81c759b4ffe6e9f9792", + "expected_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "output_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 2, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 0, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 1, + 2 + ], + "1": [ + 1, + 3 + ], + "3": [ + 0, + 2 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 5406720, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 850, + "remote_qp_number": 468, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 5406720, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 460, + "remote_qp_number": 717, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 851, + "remote_qp_number": 468, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 717, + "remote_qp_number": 461, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 469, + "remote_qp_number": 852, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 462, + "remote_qp_number": 718, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 2703360, + "payload_bytes_path_1": 2703360 + } + }, + { + "rank": 3, + "eager_samples_us": [ + 205.08800446987152, + 178.9119988679886, + 87.55200356245041 + ], + "graph_samples_us": [ + 181.59999450047812, + 88.55467041333516, + 60.80000102519989 + ], + "eager_median_us": 178.9119988679886, + "graph_median_us": 88.55467041333516, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "dcee6c3b8ab802537925c1ca8a6b9a61e8dfa599e321c78de57ab4e3492896c2", + "expected_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "output_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "5dd7020a56031ec31ca5632546126efc36de0f095e462f6a88593782f98d1c4f", + "expected_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "output_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "dcee6c3b8ab802537925c1ca8a6b9a61e8dfa599e321c78de57ab4e3492896c2", + "expected_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "output_sha256": "f8085672fe1ec140c1b4b6630e0a375ccc703cfb9a8ecc98e7c55d2f182bd954", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "510f45a03ff47ff11cb80c4ae7e33edd5fa19707bd7fd3c80e7998f71d34a173", + "expected_sha256": "a773986c394cf2f8b67267812f58785f84fd7be5ceace0f32ce3d8730abbef1a", + "output_sha256": "a773986c394cf2f8b67267812f58785f84fd7be5ceace0f32ce3d8730abbef1a", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "5dd7020a56031ec31ca5632546126efc36de0f095e462f6a88593782f98d1c4f", + "expected_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "output_sha256": "38ef226f69c13e18514f88f1836409267aa4cfb380b806a52a198756e24adc26", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 3, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 0, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 0, + 2 + ], + "1": [ + 1, + 2 + ], + "2": [ + 1, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 467, + "remote_qp_number": 852, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 460, + "remote_qp_number": 718, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 5406720, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 851, + "remote_qp_number": 469, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 5406720, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 461, + "remote_qp_number": 718, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 852, + "remote_qp_number": 469, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 2703360, + "physical_hop_payload_bytes": 2703360, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 718, + "remote_qp_number": 462, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 2703360, + "payload_bytes_path_1": 2703360 + } + } + ] + }, + { + "schema": "b12x.rocenante-virtual-diagonal-evidence/v1", + "status": "research-only", + "payload_bytes": 229376, + "warmups": 2, + "samples": 3, + "graph_operations_per_replay": 3, + "ranks": [ + { + "rank": 0, + "eager_samples_us": [ + 247.5840002298355, + 222.84799814224243, + 50.11200159788132 + ], + "graph_samples_us": [ + 63.97866706053416, + 89.32266632715861, + 61.237335205078125 + ], + "eager_median_us": 222.84799814224243, + "graph_median_us": 63.97866706053416, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "cd404c52ade92a17da6209b4cf2ee77348b5412b717f9f263965947995c676a5", + "expected_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "output_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "7caecc2288cee7679c4bf5be7d8c5dee3c5229072b85f148afae1af4c0b0e4b5", + "expected_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "output_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "cd404c52ade92a17da6209b4cf2ee77348b5412b717f9f263965947995c676a5", + "expected_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "output_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "52450971e24b57afd045cbfa9af8ce817c1edf1a1007bb858c525a14cc5b31c8", + "expected_sha256": "f2020cf5aee4a14c69ba387ab5f7ecc167433ecc88f795b63cc205821f894c29", + "output_sha256": "f2020cf5aee4a14c69ba387ab5f7ecc167433ecc88f795b63cc205821f894c29", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "7caecc2288cee7679c4bf5be7d8c5dee3c5229072b85f148afae1af4c0b0e4b5", + "expected_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "output_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 0, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 33, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "1": [ + 0, + 2 + ], + "2": [ + 0, + 3 + ], + "3": [ + 1, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 470, + "remote_qp_number": 853, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 463, + "remote_qp_number": 719, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 7569408, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 471, + "remote_qp_number": 853, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 7569408, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 720, + "remote_qp_number": 463, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 855, + "remote_qp_number": 470, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 721, + "remote_qp_number": 463, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 3784704, + "payload_bytes_path_1": 3784704 + } + }, + { + "rank": 1, + "eager_samples_us": [ + 598.8159775733948, + 125.2799928188324, + 44188.44985961914 + ], + "graph_samples_us": [ + 181.3973387082418, + 86.61333719889323, + 59.04000004132589 + ], + "eager_median_us": 598.8159775733948, + "graph_median_us": 86.61333719889323, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "9d31d0c21c09ef82790ff09044224bab4dcf862cfce6b307f590980c6cd706cf", + "expected_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "output_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "2259bf8ca07ea75b30498c193744fcb7297784c10811ada2f668ead07c696006", + "expected_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "output_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "9d31d0c21c09ef82790ff09044224bab4dcf862cfce6b307f590980c6cd706cf", + "expected_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "output_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "42e0c790effb0da9acf2a617268417e1815d7b403065c34d189f84ec21058599", + "expected_sha256": "f2020cf5aee4a14c69ba387ab5f7ecc167433ecc88f795b63cc205821f894c29", + "output_sha256": "f2020cf5aee4a14c69ba387ab5f7ecc167433ecc88f795b63cc205821f894c29", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "2259bf8ca07ea75b30498c193744fcb7297784c10811ada2f668ead07c696006", + "expected_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "output_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 1, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 33, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 1, + 3 + ], + "2": [ + 0, + 2 + ], + "3": [ + 0, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 853, + "remote_qp_number": 470, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 719, + "remote_qp_number": 463, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 471, + "remote_qp_number": 854, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 464, + "remote_qp_number": 720, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 7569408, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 472, + "remote_qp_number": 854, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 7569408, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 721, + "remote_qp_number": 464, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 2, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 3784704, + "payload_bytes_path_1": 3784704 + } + }, + { + "rank": 2, + "eager_samples_us": [ + 743.6479926109314, + 56.57599866390228, + 161.43999993801117 + ], + "graph_samples_us": [ + 118.02666385968526, + 74.15466507275899, + 65.13066589832306 + ], + "eager_median_us": 161.43999993801117, + "graph_median_us": 74.15466507275899, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "2c6e99a15b93496b40f8cc02f8177cdf5a550336950f15fa6ce0f585102f549a", + "expected_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "output_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "8ee6105aea1f6f4488d981245473db1326292aedfac99f7cef071db169f6807c", + "expected_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "output_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "2c6e99a15b93496b40f8cc02f8177cdf5a550336950f15fa6ce0f585102f549a", + "expected_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "output_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "855cc661f79e1119c78724e352f7d1ce459668530e9962b22d7b43d892865833", + "expected_sha256": "f2020cf5aee4a14c69ba387ab5f7ecc167433ecc88f795b63cc205821f894c29", + "output_sha256": "f2020cf5aee4a14c69ba387ab5f7ecc167433ecc88f795b63cc205821f894c29", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "8ee6105aea1f6f4488d981245473db1326292aedfac99f7cef071db169f6807c", + "expected_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "output_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 2, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 33, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 1, + 2 + ], + "1": [ + 1, + 3 + ], + "3": [ + 0, + 2 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 7569408, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 853, + "remote_qp_number": 471, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 7569408, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 463, + "remote_qp_number": 720, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 854, + "remote_qp_number": 471, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 720, + "remote_qp_number": 464, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 472, + "remote_qp_number": 855, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 465, + "remote_qp_number": 721, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 3784704, + "payload_bytes_path_1": 3784704 + } + }, + { + "rank": 3, + "eager_samples_us": [ + 491.4880096912384, + 124.44800138473511, + 49696.83074951172 + ], + "graph_samples_us": [ + 174.76266622543335, + 74.81599847475688, + 70.66666583220164 + ], + "eager_median_us": 491.4880096912384, + "graph_median_us": 74.81599847475688, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "54b6da6dab14daa1d0c60cda432b7128d18131c8850214039c8dda8ab3b47fc3", + "expected_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "output_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "6857eae090823b65a353ee31b96b2a5bc27174935c77dd87d8b5b8f40a8385ea", + "expected_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "output_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "54b6da6dab14daa1d0c60cda432b7128d18131c8850214039c8dda8ab3b47fc3", + "expected_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "output_sha256": "93a73648c332a1d4054183a6b8d177e63092ccb0f28cb04743c47cd32da4721c", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "e798f851c47d8673b2cab4cfa59c533604e990164d883b2393caee06d3f1eb5e", + "expected_sha256": "f2020cf5aee4a14c69ba387ab5f7ecc167433ecc88f795b63cc205821f894c29", + "output_sha256": "f2020cf5aee4a14c69ba387ab5f7ecc167433ecc88f795b63cc205821f894c29", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "6857eae090823b65a353ee31b96b2a5bc27174935c77dd87d8b5b8f40a8385ea", + "expected_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "output_sha256": "1a64b11191799028236a93dc0ffb15330e07140967c672eb0475b5e63682ea87", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 3, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 33, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 0, + 2 + ], + "1": [ + 1, + 2 + ], + "2": [ + 1, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 470, + "remote_qp_number": 855, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 463, + "remote_qp_number": 721, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 7569408, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 854, + "remote_qp_number": 472, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 7569408, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 464, + "remote_qp_number": 721, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 855, + "remote_qp_number": 472, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 3784704, + "physical_hop_payload_bytes": 3784704, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 721, + "remote_qp_number": 465, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 3784704, + "payload_bytes_path_1": 3784704 + } + } + ] + }, + { + "schema": "b12x.rocenante-virtual-diagonal-evidence/v1", + "status": "research-only", + "payload_bytes": 524288, + "warmups": 2, + "samples": 3, + "graph_operations_per_replay": 3, + "ranks": [ + { + "rank": 0, + "eager_samples_us": [ + 504.9920082092285, + 100.03200173377991, + 247.39199876785278 + ], + "graph_samples_us": [ + 196.58666849136353, + 198.27200969060263, + 114.54932888348897 + ], + "eager_median_us": 247.39199876785278, + "graph_median_us": 196.58666849136353, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "5454745e573fa5778e29b37d413f7de0e88b9dc4e8fa8d57d2e94de4852ffbbb", + "expected_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "output_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "c5b0030d4b73d266c9c673643eab9fc1dc4e86d42add28efe442304139077aad", + "expected_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "output_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "5454745e573fa5778e29b37d413f7de0e88b9dc4e8fa8d57d2e94de4852ffbbb", + "expected_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "output_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "bedbe8f86cf6239d446f137a9bd98e58556d555e60472e926749847adc960b69", + "expected_sha256": "efc2693cd87c395d15cc8e5b6414ffd3dedad9d70217a88b46599ca34aa54395", + "output_sha256": "efc2693cd87c395d15cc8e5b6414ffd3dedad9d70217a88b46599ca34aa54395", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "c5b0030d4b73d266c9c673643eab9fc1dc4e86d42add28efe442304139077aad", + "expected_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "output_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 0, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 33, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "1": [ + 0, + 2 + ], + "2": [ + 0, + 3 + ], + "3": [ + 1, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 473, + "remote_qp_number": 856, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 466, + "remote_qp_number": 722, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 17301504, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 474, + "remote_qp_number": 856, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 17301504, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 723, + "remote_qp_number": 466, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 858, + "remote_qp_number": 473, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 724, + "remote_qp_number": 466, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 8650752, + "payload_bytes_path_1": 8650752 + } + }, + { + "rank": 1, + "eager_samples_us": [ + 1352.4800539016724, + 206.496000289917, + 169.3120002746582 + ], + "graph_samples_us": [ + 112.61866490046184, + 200.91732343037924, + 23482.90252685547 + ], + "eager_median_us": 206.496000289917, + "graph_median_us": 200.91732343037924, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "f31670bd652760a135fc801b266d512d7a9011d05c6f3716807777ff5b49a1dd", + "expected_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "output_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "fc536e1d3546dd8630ac053c00be0b69b3c1444b8087e7eb9503d1fc0fb13e0b", + "expected_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "output_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "f31670bd652760a135fc801b266d512d7a9011d05c6f3716807777ff5b49a1dd", + "expected_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "output_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "cda0cfab90e33789a8d5560893aa2a659cecc52259677456582e47ccdfcb32af", + "expected_sha256": "efc2693cd87c395d15cc8e5b6414ffd3dedad9d70217a88b46599ca34aa54395", + "output_sha256": "efc2693cd87c395d15cc8e5b6414ffd3dedad9d70217a88b46599ca34aa54395", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "fc536e1d3546dd8630ac053c00be0b69b3c1444b8087e7eb9503d1fc0fb13e0b", + "expected_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "output_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 1, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 33, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 1, + 3 + ], + "2": [ + 0, + 2 + ], + "3": [ + 0, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 856, + "remote_qp_number": 473, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 722, + "remote_qp_number": 466, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 474, + "remote_qp_number": 857, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 467, + "remote_qp_number": 723, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 17301504, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 475, + "remote_qp_number": 857, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 17301504, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 724, + "remote_qp_number": 467, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 2, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 8650752, + "payload_bytes_path_1": 8650752 + } + }, + { + "rank": 2, + "eager_samples_us": [ + 285.8560085296631, + 321.4080035686493, + 117.50400066375732 + ], + "graph_samples_us": [ + 360.21331946055096, + 129.9199958642324, + 162.8053287665049 + ], + "eager_median_us": 285.8560085296631, + "graph_median_us": 162.8053287665049, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "f47b91b28cd1441d1cdece5388c65b3d168d1e908f6d7c1174dbbb45e707cb04", + "expected_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "output_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "36ecc3fa4521e727d5f750f398b915b33e833db4288e6cab028fd272d8d08180", + "expected_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "output_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "f47b91b28cd1441d1cdece5388c65b3d168d1e908f6d7c1174dbbb45e707cb04", + "expected_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "output_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "ce005080013f27240ee25d161bbbd04d3328b5988715ad979c9c1251c4256048", + "expected_sha256": "efc2693cd87c395d15cc8e5b6414ffd3dedad9d70217a88b46599ca34aa54395", + "output_sha256": "efc2693cd87c395d15cc8e5b6414ffd3dedad9d70217a88b46599ca34aa54395", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "36ecc3fa4521e727d5f750f398b915b33e833db4288e6cab028fd272d8d08180", + "expected_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "output_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 2, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 33, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 1, + 2 + ], + "1": [ + 1, + 3 + ], + "3": [ + 0, + 2 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 17301504, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 856, + "remote_qp_number": 474, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 17301504, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 466, + "remote_qp_number": 723, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 857, + "remote_qp_number": 474, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 723, + "remote_qp_number": 467, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 475, + "remote_qp_number": 858, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 3, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 468, + "remote_qp_number": 724, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 8650752, + "payload_bytes_path_1": 8650752 + } + }, + { + "rank": 3, + "eager_samples_us": [ + 1071.903944015503, + 188.6720061302185, + 5829.472064971924 + ], + "graph_samples_us": [ + 237.06666628519693, + 124.56533312797546, + 170.66667477289835 + ], + "eager_median_us": 1071.903944015503, + "graph_median_us": 170.66667477289835, + "correctness_cases": [ + { + "name": "rank-specific-index-pattern-a", + "mode": "eager", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "db58c857abdd08c9f5f79b6d74e6f9250366a29761fb62b5a2e0593a0d1e76c7", + "expected_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "output_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "passed": true + }, + { + "name": "constant-timing-reference", + "mode": "eager", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "692d6abae8f5b17bb4fb00d1c72ac0b84dc21c09552d9c78834d730fc3a920c4", + "expected_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "output_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "passed": true + }, + { + "name": "graph-input-mutation-a", + "mode": "graph-replay", + "input_formula": "((rank + 1) * ((index % 17) - 8)) + rank", + "expected_formula": "10 * ((index % 17) - 8) + 6", + "input_sha256": "db58c857abdd08c9f5f79b6d74e6f9250366a29761fb62b5a2e0593a0d1e76c7", + "expected_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "output_sha256": "510b1e3d9f4094f5df8c9eb19ab8c4c47e810dc028bd479ba4b64c86dadb3df8", + "passed": true + }, + { + "name": "graph-input-mutation-b", + "mode": "graph-replay", + "input_formula": "((4 - rank) * (((index * 5 + 3) % 19) - 9)) - rank", + "expected_formula": "10 * (((index * 5 + 3) % 19) - 9) - 6", + "input_sha256": "1808a13e28838bc253caf156b3c19c85889232790076f5f4a33eea2f2b40b9b8", + "expected_sha256": "efc2693cd87c395d15cc8e5b6414ffd3dedad9d70217a88b46599ca34aa54395", + "output_sha256": "efc2693cd87c395d15cc8e5b6414ffd3dedad9d70217a88b46599ca34aa54395", + "passed": true + }, + { + "name": "constant-timing-output", + "mode": "graph-replay", + "input_formula": "rank + 1", + "expected_formula": "10", + "input_sha256": "692d6abae8f5b17bb4fb00d1c72ac0b84dc21c09552d9c78834d730fc3a920c4", + "expected_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "output_sha256": "33fc9307e1ef3253c401b96f932255aab1a8d3d7a317168e9eded52d2dc7bc37", + "passed": true + } + ], + "stats": { + "world_size": 4, + "rank": 3, + "hcas": [ + "rocep1s0f0", + "rocep1s0f1", + "roceP2p1s0f0", + "roceP2p1s0f1" + ], + "max_size": 2097152, + "max_gather_bytes": 2097152, + "slot_bytes": 2097152, + "epoch": 33, + "error_seq": 0, + "error_peer": 0, + "ctrl_seq": 33, + "spin_limit": 20000000, + "opposite_paths": 2, + "ops_posted": 33, + "writes_completed": 198, + "last_seq": 33, + "two_wave_activations": 33, + "two_wave_threshold_bytes": 196608, + "wave_mode": "two", + "peer_hca": { + "0": [ + 0, + 2 + ], + "1": [ + 1, + 2 + ], + "2": [ + 1, + 3 + ] + } + }, + "path_counters": [ + { + "peer_rank": 0, + "path_index": 0, + "device": "rocep1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 473, + "remote_qp_number": 858, + "local_hca_index": 0, + "remote_hca_index": 1, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 0, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 466, + "remote_qp_number": 724, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 17301504, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 857, + "remote_qp_number": 475, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 1, + "path_index": 1, + "device": "roceP2p1s0f0", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 17301504, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 467, + "remote_qp_number": 724, + "local_hca_index": 2, + "remote_hca_index": 3, + "physical_hops": 2, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 0, + "device": "rocep1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 858, + "remote_qp_number": 475, + "local_hca_index": 1, + "remote_hca_index": 0, + "physical_hops": 1, + "retries": null, + "retry_events": null + }, + { + "peer_rank": 2, + "path_index": 1, + "device": "roceP2p1s0f1", + "payload_writes": 33, + "payload_bytes": 8650752, + "physical_hop_payload_bytes": 8650752, + "flag_writes": 33, + "send_completions": 33, + "completion_errors": 0, + "qp_number": 724, + "remote_qp_number": 468, + "local_hca_index": 3, + "remote_hca_index": 2, + "physical_hops": 1, + "retries": null, + "retry_events": null + } + ], + "expected_counters": { + "operations_per_rank": 33, + "paths_per_rank": 6, + "flags_per_path": 33, + "payload_bytes_path_0": 8650752, + "payload_bytes_path_1": 8650752 + } + } + ] + } + ], + "streams": [ + { + "checks": [ + { + "rank": 0, + "payload_bytes": 32768, + "alternating_stream_calls": 16, + "misaligned_input_output": true, + "changed_input_graph_replays": 2, + "capture_stream_rejected": true, + "passed": true + }, + { + "rank": 1, + "payload_bytes": 32768, + "alternating_stream_calls": 16, + "misaligned_input_output": true, + "changed_input_graph_replays": 2, + "capture_stream_rejected": true, + "passed": true + }, + { + "rank": 2, + "payload_bytes": 32768, + "alternating_stream_calls": 16, + "misaligned_input_output": true, + "changed_input_graph_replays": 2, + "capture_stream_rejected": true, + "passed": true + }, + { + "rank": 3, + "payload_bytes": 32768, + "alternating_stream_calls": 16, + "misaligned_input_output": true, + "changed_input_graph_replays": 2, + "capture_stream_rejected": true, + "passed": true + } + ], + "passed": true + }, + { + "checks": [ + { + "rank": 0, + "payload_bytes": 524288, + "alternating_stream_calls": 16, + "misaligned_input_output": true, + "changed_input_graph_replays": 2, + "capture_stream_rejected": true, + "passed": true + }, + { + "rank": 1, + "payload_bytes": 524288, + "alternating_stream_calls": 16, + "misaligned_input_output": true, + "changed_input_graph_replays": 2, + "capture_stream_rejected": true, + "passed": true + }, + { + "rank": 2, + "payload_bytes": 524288, + "alternating_stream_calls": 16, + "misaligned_input_output": true, + "changed_input_graph_replays": 2, + "capture_stream_rejected": true, + "passed": true + }, + { + "rank": 3, + "payload_bytes": 524288, + "alternating_stream_calls": 16, + "misaligned_input_output": true, + "changed_input_graph_replays": 2, + "capture_stream_rejected": true, + "passed": true + } + ], + "passed": true + } + ], + "startup": { + "ready": true, + "seconds": 437.3279999999995 + }, + "restart": { + "ready": true, + "seconds": 289.6409999999996 + }, + "idle_rank_loss": { + "injected_rank": 3, + "passed": true, + "all_model_ranks_stopped_seconds": 10.063000000000102, + "scope": "No in-flight request, link loss, or stalled GPU collective was injected." + }, + "persistent_cache": { + "prompt_sha256": "3d2bc5228895566b1497e6f35f6c5aa051685f99438f27226134dfcfab15c277", + "prompt_tokens": 27274, + "external_hit_tokens": 26624, + "all_rank_restore_tokens": [ + 26624, + 26624, + 26624, + 26624 + ], + "semantic_passed": true, + "expected_answer": "cobalt orchard lantern", + "cold_request_seconds": 11.85899999999856, + "restored_request_seconds": 2.9219999999986612, + "persistent_restore_proven": true + }, + "matrices": [ + { + "receipt_sha256": "49bcec98b5e39d10a4493a1056c9ac7589a9caeba47c0f64fada18f586f7f1fa", + "receipt_name": "combined-matrix.json", + "metadata": { + "version": "0.4.32", + "model": "glm-5.3-flash-spark", + "temperature": 1.0, + "max_tokens": 2048, + "duration_per_test": 20.0, + "context_lengths": [ + 8192, + 32768, + 65536 + ], + "concurrency_levels": [ + 1, + 2, + 4, + 8, + 12, + 16 + ] + }, + "prefill": { + "8192": { + "tok_per_sec": 2648.0, + "ttft_seconds": 3.094, + "method": "integrated_scout", + "samples": 1 + }, + "32768": { + "tok_per_sec": 2745.0, + "ttft_seconds": 11.938, + "method": "integrated_scout", + "samples": 1 + }, + "65536": { + "tok_per_sec": 2687.0, + "ttft_seconds": 24.391, + "method": "integrated_scout", + "samples": 1 + }, + "131072": { + "tok_per_sec": 2750.0, + "ttft_seconds": 47.656, + "method": "scout_only", + "samples": 1 + } + }, + "cells": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 50.728629375534254, + "server_steps_per_s": 18.578797135560915, + "server_spec_accept_length": 2.730458221024259, + "num_errors": 0 + }, + { + "context_tokens": 32768, + "concurrency": 1, + "aggregate_tps": 45.68654923939065, + "server_steps_per_s": 17.163730984787506, + "server_spec_accept_length": 2.6618075801749272, + "num_errors": 0 + }, + { + "context_tokens": 65536, + "concurrency": 1, + "aggregate_tps": 49.53715286464704, + "server_steps_per_s": 18.864148111082763, + "server_spec_accept_length": 2.6259946949602124, + "num_errors": 0 + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 74.5752518418301, + "server_steps_per_s": 28.29061162364332, + "server_spec_accept_length": 2.636042402826855, + "num_errors": 0 + }, + { + "context_tokens": 8192, + "concurrency": 4, + "aggregate_tps": 117.88841631223075, + "server_steps_per_s": 42.03152364273083, + "server_spec_accept_length": 2.8047619047619046, + "num_errors": 0 + }, + { + "context_tokens": 8192, + "concurrency": 8, + "aggregate_tps": 181.0396634615316, + "server_steps_per_s": 64.50320512820268, + "server_spec_accept_length": 2.8066770186335406, + "num_errors": 0 + }, + { + "context_tokens": 8192, + "concurrency": 12, + "aggregate_tps": 200.85, + "server_steps_per_s": 71.76774193548387, + "server_spec_accept_length": 2.798611111111111, + "num_errors": 0 + }, + { + "context_tokens": 8192, + "concurrency": 16, + "aggregate_tps": 230.49165539017244, + "server_steps_per_s": 83.39598055430461, + "server_spec_accept_length": 2.7638221153846154, + "num_errors": 0 + }, + { + "context_tokens": 32768, + "concurrency": 2, + "aggregate_tps": 78.83526286774108, + "server_steps_per_s": 28.968074976194753, + "server_spec_accept_length": 2.7214532871972317, + "num_errors": 0 + }, + { + "context_tokens": 32768, + "concurrency": 4, + "aggregate_tps": 120.3, + "server_steps_per_s": 44.0, + "server_spec_accept_length": 2.7340909090909093, + "num_errors": 0 + }, + { + "context_tokens": 32768, + "concurrency": 8, + "aggregate_tps": 178.8907866974868, + "server_steps_per_s": 63.90033155832721, + "server_spec_accept_length": 2.7995283018867925, + "num_errors": 0 + }, + { + "context_tokens": 32768, + "concurrency": 12, + "aggregate_tps": 204.42433383610452, + "server_steps_per_s": 73.60482654600517, + "server_spec_accept_length": 2.777322404371585, + "num_errors": 0 + }, + { + "context_tokens": 32768, + "concurrency": 16, + "aggregate_tps": 229.71952263456936, + "server_steps_per_s": 82.17936451986348, + "server_spec_accept_length": 2.795343137254902, + "num_errors": 0 + }, + { + "context_tokens": 65536, + "concurrency": 2, + "aggregate_tps": 72.30784627702025, + "server_steps_per_s": 28.82305844675686, + "server_spec_accept_length": 2.5086805555555554, + "num_errors": 0 + }, + { + "context_tokens": 65536, + "concurrency": 4, + "aggregate_tps": 122.33962642094659, + "server_steps_per_s": 43.267063949119056, + "server_spec_accept_length": 2.8275462962962963, + "num_errors": 0 + }, + { + "context_tokens": 65536, + "concurrency": 8, + "aggregate_tps": 170.77808356268258, + "server_steps_per_s": 62.046534901179726, + "server_spec_accept_length": 2.7524193548387097, + "num_errors": 0 + }, + { + "context_tokens": 65536, + "concurrency": 12, + "aggregate_tps": 205.38364779874215, + "server_steps_per_s": 73.05660377358491, + "server_spec_accept_length": 2.8112947658402203, + "num_errors": 0 + }, + { + "context_tokens": 65536, + "concurrency": 16, + "aggregate_tps": 235.47508573734683, + "server_steps_per_s": 82.30784748840223, + "server_spec_accept_length": 2.860906862745098, + "num_errors": 0 + } + ] + }, + { + "receipt_sha256": "a5cd0abe72a3bfd7035e9019fc4152ad9279bb6cd6edc4795c69d081c3b6882b", + "receipt_name": "combined-confirmation-1.json", + "metadata": { + "version": "0.4.32", + "model": "glm-5.3-flash-spark", + "temperature": 1.0, + "max_tokens": 2048, + "duration_per_test": 20.0, + "context_lengths": [ + 8192, + 65536 + ], + "concurrency_levels": [ + 1, + 2 + ] + }, + "prefill": { + "8192": { + "tok_per_sec": 2661.0, + "ttft_seconds": 3.078, + "method": "integrated_scout", + "samples": 1 + }, + "65536": { + "tok_per_sec": 2770.0, + "ttft_seconds": 23.656, + "method": "integrated_scout", + "samples": 1 + } + }, + "cells": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 49.47668886774713, + "server_steps_per_s": 18.678952376183886, + "server_spec_accept_length": 2.648793565683646, + "num_errors": 0 + }, + { + "context_tokens": 65536, + "concurrency": 1, + "aggregate_tps": 53.23251039110365, + "server_steps_per_s": 18.85837762683897, + "server_spec_accept_length": 2.822751322751323, + "num_errors": 0 + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 76.66132906324916, + "server_steps_per_s": 29.485126562788135, + "server_spec_accept_length": 2.6, + "num_errors": 0 + }, + { + "context_tokens": 65536, + "concurrency": 2, + "aggregate_tps": 79.2456615508071, + "server_steps_per_s": 29.692045340555573, + "server_spec_accept_length": 2.668918918918919, + "num_errors": 0 + } + ] + }, + { + "receipt_sha256": "1b07b7de5e927e1d755e8f3e33aa63c0765508e358f3bffc73a3af55e5f94006", + "receipt_name": "combined-confirmation-2.json", + "metadata": { + "version": "0.4.32", + "model": "glm-5.3-flash-spark", + "temperature": 1.0, + "max_tokens": 2048, + "duration_per_test": 20.0, + "context_lengths": [ + 8192, + 65536 + ], + "concurrency_levels": [ + 1, + 2 + ] + }, + "prefill": { + "8192": { + "tok_per_sec": 2661.0, + "ttft_seconds": 3.078, + "method": "integrated_scout", + "samples": 1 + }, + "65536": { + "tok_per_sec": 2767.0, + "ttft_seconds": 23.688, + "method": "integrated_scout", + "samples": 1 + } + }, + "cells": [ + { + "context_tokens": 8192, + "concurrency": 1, + "aggregate_tps": 48.25, + "server_steps_per_s": 18.950000000000003, + "server_spec_accept_length": 2.5461741424802113, + "num_errors": 0 + }, + { + "context_tokens": 65536, + "concurrency": 1, + "aggregate_tps": 50.89071257005508, + "server_steps_per_s": 18.81505204163295, + "server_spec_accept_length": 2.704787234042553, + "num_errors": 0 + }, + { + "context_tokens": 8192, + "concurrency": 2, + "aggregate_tps": 75.48400040124346, + "server_steps_per_s": 29.09017955662539, + "server_spec_accept_length": 2.594827586206897, + "num_errors": 0 + }, + { + "context_tokens": 65536, + "concurrency": 2, + "aggregate_tps": 80.048149262714, + "server_steps_per_s": 28.799334150732836, + "server_spec_accept_length": 2.779513888888889, + "num_errors": 0 + } + ] + } + ], + "raw_receipt_sha256": { + "image-c139f3670-receipt.json": "91e552e5274716d568f131b515a9f76a663b66e6cf3a72528675d3209ccd3857", + "model-ready.json": "3fa1942c036ea40fd28861eafd9a0fa96c1df3ee397dbbf4ec8b0094911d4171", + "model-ready-after-restart.json": "c72c18eeb15551cfb04f5b6555602efa9444e7ad144230e2665ad0cabcdb0622", + "idle-rank-loss.json": "af4d0cb0ab54b318b3fb58bdca45e3fba3bae5e6682f36d42e41e55cabde1d30", + "cache-publication.log": "630ee5413cc19bf4a7225a62c08356c8c5ace826e7e444ca89af097c75b3181f", + "cache-restore.log": "d18b34f62b45bfaf0424ccfcf1f25adb91a60ae16e3d5b48627efa69074ffd91", + "benchmark-command.json": "44f4e119f787e4a2086398dda5ce7164157040414c1b593248370f8d56af0a81" + }, + "limits": [ + "Idle rank-loss containment does not establish in-flight GPU failure containment or resolve upstream RoCEnante #313.", + "One persistent recall fixture does not establish all cache boundaries or model accuracy.", + "Serving measurements are observations, not a guaranteed speedup." + ] +} diff --git a/performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md b/performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md new file mode 100644 index 00000000..ad51d179 --- /dev/null +++ b/performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md @@ -0,0 +1,63 @@ +# GLM native-MTP3 compute and stream-safety image validation + +Status: **qualified** for the bounded checks listed below. The serving profile +remains **research-only**. + +## Configuration + +Four NVIDIA DGX Sparks serve GLM-5.3-Flash-NVFP4-Spark with TP4/DCP4, native +MTP3, an NVFP4/BF16 proposal head, a BF16 verifier head, 24 GiB FP8 KV per +rank, an 8,192-token scheduling limit, and 16 sequences. SIRCL and RoCEnante +use the hardware-forwarded physical ring. SparkCache uses the profile's +compute-specific namespace. + +- Tested and published config-image ID: + `sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f`. +- Registry manifest: + `sha256:67dc0ae453baaae6831ccec1d259b4ef8b236a8b0dc9f747d901b95c66ec1987`. +- Compute source lock: + `139f36701e0e47f45bf99fba2cc2fa59b417f2ee801dad3a064455d5b464a459`. +- Transport manifest: + `69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500`. + +The [numeric record](spark-mtp3-compute-stream-safety-20260906.json) includes +per-rank collective evidence, stream checks, all serving cells, source hashes, +and cache measurements. Host startup used the memory gate proposed in +[PR 222](https://github.com/FujitsuPolycom/sparkring/pull/222). + +## Results + +| Check | Result | +|---|---| +| Installed compute package comparison | 4,891 vLLM, 385 B12X, and 150 SparkCache files match the compute-tested image; selected environment values match | +| Four-rank native all-reduce | Exact BF16 results at 4, 20, 28, and 64 token rows; graph input mutations and per-QP completion counters pass | +| GPU stream safety | At 4 and 64 rows, 16 alternating-stream calls with misaligned input/output, two changed-input graph replays, and second-stream capture rejection pass | +| Full serving matrix | 18 cells complete without errors, capacity limitation, underfilling, or warm-up timeout | +| Focused repetitions | Two four-cell 8K/64K C1/C2 runs complete cleanly; 64K prefill measures 2,770 and 2,767 tok/s | +| Idle model-rank loss | Deliberately stopping rank 3 causes all model ranks to stop within 10.063 seconds | +| Recovery and restart | Same image and container identities pass managed recovery, startup memory checks, and four-rank readiness | +| Persistent prefix restore | Correct phrase recall after restart, 26,624 external-hit tokens, matching restore logs on all four ranks | +| Public image access | Anonymous manifest/config verification and Docker pull pass; pull reused local image layers | + +The serving harness is version 0.4.32, pinned by source hash. It uses +temperature 1, 2,048 maximum output tokens, 20-second cells, 8K/32K/64K +contexts and C1/C2/C4/C8/C12/C16. The full matrix's initial 64K prefill scout +measured 2,687 tok/s; that lower result did not repeat in the two focused runs. +Individual decode cells vary. These results establish a functioning composed +image, not a guaranteed performance improvement over every tested configuration. + +The persistent-cache fixture contains 27,274 prompt tokens. Both requests +returned `cobalt orchard lantern` with a normal stop. Full request time was +11.859 seconds cold and 2.922 seconds after restart; restore logs reported +26,624 tokens on every rank, and the engine reported 26,624 external-hit +tokens. This proves persistent restoration for that fixture rather than merely +fast GPU-prefix reuse within one process. + +## Scope + +The rank-loss test used an idle model. It does not establish containment of +an in-flight stalled GPU collective or resolve upstream RoCEnante issue #313. +One recall fixture does not qualify every cache boundary, concurrent cache +workload, or general model accuracy. The recorded checks do not exercise +unattended high availability. Watchdog and memory-protection thresholds were +not relaxed. All four model containers were healthy after restoration. diff --git a/recipes/glm53-spark-mtp3-managed-mesh-tp4.json b/recipes/glm53-spark-mtp3-managed-mesh-tp4.json index 7d289a7f..9d9006eb 100644 --- a/recipes/glm53-spark-mtp3-managed-mesh-tp4.json +++ b/recipes/glm53-spark-mtp3-managed-mesh-tp4.json @@ -35,9 +35,9 @@ "public_image_contract": "runtime/glm53-spark-mtp3-mesh/public-image.json", "image_receipt": "runtime/glm53-spark-mtp3-mesh/image-receipt.json", "compute_contract": "runtime/glm53-spark-mtp3-mesh/compute/source-lock.json", - "compute_contract_sha256": "2a444f7c1ad4319f64afbffb16403491a4863934372cf818f18c510d58c0d00e", - "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:b65d427f9be49c97d57e404ad1a6118769c1119df876a8944c1f186a6b380c5d", - "image_id": "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", + "compute_contract_sha256": "139f36701e0e47f45bf99fba2cc2fa59b417f2ee801dad3a064455d5b464a459", + "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:67dc0ae453baaae6831ccec1d259b4ef8b236a8b0dc9f747d901b95c66ec1987", + "image_id": "sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f", "site_template": "runtime/glm53-spark-mtp3-mesh/site.example.json", "fabric_template": "runtime/glm53-spark-mtp3-mesh/fabric.example.json", "renderer": "runtime/glm53-spark-mtp3-mesh/profile.py", @@ -64,7 +64,24 @@ "moe_backend": "B12X", "linear_backend": "B12X", "kda_prefill_backend": "b12x", - "cudagraph_capture_sizes": [4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64], + "cudagraph_capture_sizes": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 36, + 40, + 44, + 48, + 52, + 56, + 60, + 64 + ], "speculation": { "method": "mtp", "num_speculative_tokens": 3, @@ -100,13 +117,19 @@ "async_page_capture": true, "capture_slots_per_rank": 2, "capture_slot_bytes": 3221225472, - "namespace": "glm53-spark-df116c4f-mtp3-nvfp4-a16-b58f34ea-mesh4204fabc-tail-cow-v2", + "namespace": "glm53-spark-df116c4f-mtp3-nvfp4-a16-c139f3670-mesh69313e19-tail-cow-v2", "identity_contract": "runtime/glm53-spark-mtp3-mesh/pins.json#/cache_identity" }, "transport": { "bundle_manifest_sha256": "69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500", "routing_contract": "spark_transport/experiments/glm53_rocenante_overlay/overlay_contract.json", - "captured_sircl_query_rows": [16, 20, 24, 28, 32], + "captured_sircl_query_rows": [ + 16, + 20, + 24, + 28, + 32 + ], "large_eager_prefill": "dual-rail fused SIRCL", "small_collectives": "RoCEnante on admitted rows; other captured rows use SIRCL", "fallback": "patched NCCL", @@ -118,16 +141,14 @@ "management_guide": "runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md" }, "evidence": { - "record": "performance/records/glm53-flash/spark-mtp3-managed-mesh-functional-20260905.md", + "record": "performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md", "proposal_head_performance_record": "performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md", "status": "research-only", - "scope": "Proposal-head performance was measured on serving image 04d5a35b, identified by the full digest in the proposal-head record. Managed lifecycle and persistent-recall results qualify only the image digests and cache namespaces in their linked functional records.", + "scope": "Exact-image native all-reduce, selected GPU stream invariants, bounded serving matrix, idle rank-loss shutdown, restart and one persistent-prefix recall.", "limitations": [ - "This opt-in profile does not replace the recommended DFlash/SIRCL profile.", - "The published compute image has build/content equivalence to the tested private image but requires fresh native, startup, restart and persistent-cache receipts before those qualifications transfer.", - "No host-reboot, prolonged-soak, unattended high-availability or general cache/model-quality qualification is claimed.", - "Fault detection has a bounded observation window; partial streamed output can precede shutdown.", - "Readiness warmup covers temperature-one sampling with thinking disabled, not all mixed-prefill/decode shapes." + "Idle rank-loss containment does not establish in-flight GPU failure containment or resolve upstream RoCEnante #313.", + "One persistent recall fixture does not establish all cache boundaries or model accuracy.", + "Serving measurements are observations, not a guaranteed speedup." ] } } diff --git a/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md b/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md index d027596f..8dbb8229 100644 --- a/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md +++ b/runtime/glm53-spark-mtp3-mesh/IMAGE_BUILD.md @@ -3,7 +3,8 @@ Status: **research-only**. This child image packages the complete compute and transport composition used by the GLM-5.3 Spark native-MTP3 profile. On top of the pinned parent, it installs CUDA 13.3, the checksum-bound vLLM metadata and -proposal-head patch, complete B12X revision `b58f34ea`, the transport bundle, +proposal-head and loader/RNG patches, B12X revision `ef308bac` with the +source-checked top-k selector, the transport bundle, the managed marker, and the readiness helper. The proposal head uses runtime NVFP4 weights with BF16 activations; the target/verifier head retains its BF16 checkpoint representation. SparkCache and patched NCCL remain inherited. @@ -30,19 +31,19 @@ to its Linux/ARM64 config-image identity. Pull before using its local image ID: ```bash set -euo pipefail -mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:b65d427f9be49c97d57e404ad1a6118769c1119df876a8944c1f186a6b380c5d' -mtp_image_id='sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766' +mtp_image='ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:67dc0ae453baaae6831ccec1d259b4ef8b236a8b0dc9f747d901b95c66ec1987' +mtp_image_id='sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f' docker pull "$mtp_image" test "$(docker image inspect "$mtp_image" --format '{{.Id}}')" = "$mtp_image_id" ``` The immutable reference is also published as tag -`glm53-spark-mtp3-stream69313e19`; use the digest above for deployment. +`glm53-spark-mtp3-c139f3670-mesh69313e19`; use the digest above for deployment. The [compute-image equivalence record](compute-image-equivalence.json) verifies that all 4,891 vLLM, 385 B12X, and 150 SparkCache package files and the selected -environment match tested private image `sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74`. -This is build/content equivalence, not a fresh serving, restart, or persistent- -cache qualification of the public image ID. +environment match compute-tested image `sha256:3b4768e5ba31cadcc882dffa06d7b667af44abdf157d5c11b7ac7fe962e80c43`. +The published config-image ID also passed its own +[GPU, serving, restart, and persistent-cache checks](../../performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md). Use the repository's [content receipt](image-receipt.json) for the renderer, installer, and native qualification runner. This default deployment requires @@ -143,7 +144,7 @@ Image construction has networking disabled. Verification runs with no host device mounts, no Linux capabilities, no network, a read-only root filesystem, two CPUs, and a 2 GiB memory limit. It checks the complete parent layer prefix, package source and native-library hashes, CUDA 13.3 components, the complete -B12X `b58f34ea` package, vLLM input/output hashes and proposal-head environment, +B12X `ef308bac` package with selector overrides, vLLM input/output hashes and proposal-head environment, bundle hashes, Python syntax, readiness warmup helper hash and temperature, lazy RoCEnante import, and marker linkage. CUDA must remain uninitialized. diff --git a/runtime/glm53-spark-mtp3-mesh/README.md b/runtime/glm53-spark-mtp3-mesh/README.md index b4df3a82..4748f7ec 100644 --- a/runtime/glm53-spark-mtp3-mesh/README.md +++ b/runtime/glm53-spark-mtp3-mesh/README.md @@ -1,15 +1,10 @@ # GLM-5.3 Flash Spark with native MTP3 and hardware-forwarded mesh -Status: **research-only**. Bundle composition, site rendering, managed -host-fabric installation/supervision, and CPU checks are **implemented**. -The [managed functional record](../../performance/records/glm53-flash/spark-mtp3-managed-mesh-functional-20260905.md) -qualifies bounded installer, policy-scoped fault/recovery, post-recovery -readiness, and one persistent-cache recall case. Broader coverage remains -unqualified. The -[sampling-warmup image functional record](../../performance/records/glm53-flash/spark-mtp3-mesh-temperature-one-functional-20260905.md) -qualifies bounded native checks, four-rank startup/restart, and one persistent -recall restoration for its exact image. Broader cache/workload coverage and -failure containment remain unqualified. +Status: **research-only** profile. Bundle composition, site rendering, managed +host services, and CPU checks are **implemented**. The published image has +[qualified bounded GPU, serving, restart, and persistent-cache checks](../../performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md). +The record identifies the exact image, measurements, and scope; it does not +establish in-flight collective failure containment or unattended availability. This profile serves the `GLM-5.3-Flash-NVFP4-Spark` checkpoint with its built-in multi-token predictor at depth three. The predictor uses a separate runtime- @@ -74,10 +69,10 @@ The published image's vLLM, B12X, and SparkCache package file and selected environment entry to the serving image identified in that record. This establishes compute-package content equivalence, not end-to-end performance equivalence: the mounted -transport differs outside that comparison. Throughput belongs to measured -serving image `04d5a35b`; the full digest is in the equivalence record. Native -transport, model startup, restart, and persistent-cache checks remain -image-specific and have not been repeated on published image `69c794bf`. +transport differs outside that comparison. The published image's separate +[runtime validation](../../performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md) +includes an 18-cell matrix, two focused repetitions, native and GPU stream +checks, an idle rank-loss test, restart, and a verified persistent-prefix restore. The separate [Estonia accuracy record](../../performance/records/glm53-flash/spark-mtp3-country-recall-20260905.md) reports **30/30 correct** at C8 on one repeated 133,208-token prompt, no @@ -89,7 +84,8 @@ The [long-context needle hunt](../../performance/records/glm53-flash/spark-mtp3- passed **4/4** exact-value, revision, and cross-reference checks, reaching **507,367 actual prompt tokens** on serving image `sha256:26273b8e358df139ae913610a5d43084ff0fd08aafe282ef633a3bc74afefe47`, -as recorded in that report. It is not a measurement of image `69c794bf`. +as recorded in that report. It does not identify the image pinned by +`public-image.json`. ## Composition @@ -97,7 +93,7 @@ as recorded in that report. It is not a measurement of image `69c794bf`. |---|---| | Model, MTP depth, graph shapes, mesh bundle, marker identity, cache identity | [`pins.json`](pins.json) | | Linux/ARM64 parent image, SparkCache, and native SIRCL | [`../glm53-flash-jj-r8-gb10/pins.json`](../glm53-flash-jj-r8-gb10/pins.json) | -| CUDA 13.3, GLM metadata port, complete B12X `b58f34ea`, and runtime-NVFP4/BF16 proposal head | [`pins.json`](pins.json), [`IMAGE_BUILD.md`](IMAGE_BUILD.md) | +| CUDA 13.3, GLM metadata/loader/RNG integration, B12X scale sharing and selector overrides, NVFP4/BF16 proposal head | [`pins.json`](pins.json), [`IMAGE_BUILD.md`](IMAGE_BUILD.md) | | Topology and rank-local filesystem inputs | [`site.example.json`](site.example.json), [`fabric.example.json`](fabric.example.json) | | Source-bound collective dispatch and health checks | [`glm53_rocenante_overlay`](../../spark_transport/experiments/glm53_rocenante_overlay/README.md) | | Hardware-forwarding plan and native source marker | [`cx7_hairpin_diagonal`](../../spark_transport/experiments/cx7_hairpin_diagonal/README.md) | @@ -105,7 +101,8 @@ as recorded in that report. It is not a measurement of image `69c794bf`. The managed profile requires the [published child image](IMAGE_BUILD.md). It retains the parent runtime while adding CUDA 13.3, the uniform native-MTP3 -metadata port, complete B12X revision `b58f34ea`, the runtime-NVFP4/BF16 +metadata and loader/RNG integration, B12X revision `ef308bac` with selector +overrides, the runtime-NVFP4/BF16 proposal head, the verified transport bundle, the managed source marker, and the temperature-one readiness helper. The verifier remains BF16. Pull the immutable reference in [public-image.json](public-image.json) @@ -175,8 +172,8 @@ serving lifecycle. Use authenticated managed readiness for serving. Native MTP uses the target checkpoint as the draft identity. The profile sets SparkCache's `draft_policy=separate` because that describes the registered state layout; it does not request an external model. A dedicated namespace -includes `mtp3-nvfp4-a16-b58f34ea` so the NVFP4-proposal-head compute -composition cannot restore shared-BF16-head or external-DFlash entries. +binds compute lock `139f3670` and transport bundle `69313e19` so the +NVFP4-proposal-head composition cannot restore entries from another computation. Do not relabel those entries to avoid cache misses. Persistent restore under the native-MTP identity requires its own qualification. diff --git a/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json b/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json index 02579b98..0e255476 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json +++ b/runtime/glm53-spark-mtp3-mesh/compute-image-equivalence.json @@ -1,8 +1,8 @@ { "schema": "sparkring-compute-image-equivalence/v1", "checks_passed": true, - "tested_serving_image": "sha256:04d5a35b03e99f68c37a05514d221988a3eb70a5b8fdcfa859025ca1cbc25e74", - "published_image": "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", + "tested_serving_image": "sha256:3b4768e5ba31cadcc882dffa06d7b667af44abdf157d5c11b7ac7fe962e80c43", + "published_image": "sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f", "package_files": { "b12x": 385, "sparkcache": 150, @@ -10,8 +10,6 @@ }, "package_files_identical": true, "selected_environment_identical": true, - "tested_snapshot_sha256": "54a975edcb68df32b6520f7070b21e0b88e280f847d5039c81bb1f2f99f7cc51", - "published_snapshot_sha256": "54a975edcb68df32b6520f7070b21e0b88e280f847d5039c81bb1f2f99f7cc51", "environment": { "CUDA_HOME": "/opt/cuda-13.3", "SPARKRING_WARMUP_TEMPERATURE": "1", @@ -22,5 +20,8 @@ "VLLM_MTP_NVFP4_LM_HEAD": "1", "VLLM_MXFP8_LM_HEAD": "0" }, - "scope": "Installed compute package files excluding Python bytecode and selected environment. The mounted RoCEnante transport overlay additionally contains capture-guard and shared-staging ordering fixes; it is not included in compute package equivalence. Public image construction and native-marker verification passed. Full-model serving was measured on the tested serving image; no separate four-rank serving run is claimed for this published image." + "tested_snapshot_sha256": "7d2413eea1f2da3163f40f2a7cf937aeacb1bca44bdc5477fac87b94594b92b0", + "published_snapshot_sha256": "7d2413eea1f2da3163f40f2a7cf937aeacb1bca44bdc5477fac87b94594b92b0", + "scope": "All installed compute package files except Python bytecode. Mounted transport differs and is validated separately in the linked exact-image record.", + "functional_record": "../../performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md" } diff --git a/runtime/glm53-spark-mtp3-mesh/compute/README.md b/runtime/glm53-spark-mtp3-mesh/compute/README.md index 81d01f4e..8eab2991 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/README.md +++ b/runtime/glm53-spark-mtp3-mesh/compute/README.md @@ -48,7 +48,7 @@ revision `3512b066e7796128c0c380ccc558182960f2f0ea`, with dense-kernel integrati from revision `a8c796f3af74106b2d8d441e9ec54588936a5388`; vLLM is licensed under Apache License 2.0. -B12X source archives use LF endings. Source preparation converts Python, C, -and package Markdown files to CRLF to reproduce the package hashes in -`source-lock.json`. Compressed profile data retain the archive bytes. This byte-level +B12X source archives use LF endings. Source preparation converts Python and C +files to CRLF to reproduce the installed package hashes in `source-lock.json`. +Markdown and compressed profile data retain the archive bytes. This byte-level contract makes package-content verification independent of checkout settings. diff --git a/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json b/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json index b01e3efe..8ff0b729 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json +++ b/runtime/glm53-spark-mtp3-mesh/compute/source-lock.json @@ -141,12 +141,11 @@ "tree": "dcf039e5e754136275835ea997e6b9abbb6b15ae", "archive_url": "https://github.com/local-inference-lab/b12x/archive/ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301.tar.gz", "archive_sha256": "029a6047d80f759e964eb302803df3dc5d2ee324b0a0725d38694b5057ed69a6", - "package_files_sha256": "9c33a4ef28eb83c81af446e3c02672b7e3318dc434f16a54f304fccf7c2de0f4", + "package_files_sha256": "a212d449381174fabfe681c48cde16b22e3b9f9fa7942ab071b0c7384dc78abd", "installed_text_line_endings": "crlf", "normalized_text_suffixes": [ ".c", - ".py", - ".md" + ".py" ], "overrides": { "archive": "b12x-selector-files.tar.gz", diff --git a/runtime/glm53-spark-mtp3-mesh/image-receipt.json b/runtime/glm53-spark-mtp3-mesh/image-receipt.json index 9d0fcc8b..be2b9b5d 100644 --- a/runtime/glm53-spark-mtp3-mesh/image-receipt.json +++ b/runtime/glm53-spark-mtp3-mesh/image-receipt.json @@ -2,19 +2,19 @@ "added_layers": 8, "bundle_manifest_sha256": "69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500", "checks_passed": true, - "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache:glm53-spark-mtp3-stream69313e19", - "image_id": "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", - "image_reference": "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", - "image_size_bytes": 22107699822, + "image": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:67dc0ae453baaae6831ccec1d259b4ef8b236a8b0dc9f747d901b95c66ec1987", + "image_id": "sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f", + "image_reference": "sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f", + "image_size_bytes": 22108005586, "inside_image": { - "b12x_commit": "b58f34eaf978277621efced6678e6713fd7122e4", + "b12x_commit": "ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301", "bundle_files": 28, "bundle_manifest_sha256": "69313e19e881ec93e9ed3bd150d2f24fc6b444488ac729a69f45d038e2243500", "checks_passed": true, "compute": { "b12x_files": 385, - "b12x_revision": "b58f34eaf978277621efced6678e6713fd7122e4", - "b12x_tree": "7637fe5fb4d88882e0d18cdacc68c493f478499d", + "b12x_revision": "ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301", + "b12x_tree": "dcf039e5e754136275835ea997e6b9abbb6b15ae", "cuda_version": "13.3", "environment": { "CUDA_HOME": "/opt/cuda-13.3", @@ -26,9 +26,9 @@ "VLLM_MXFP8_LM_HEAD": "0" }, "proposal_head_nvfp4": true, - "source_lock_sha256": "2a444f7c1ad4319f64afbffb16403491a4863934372cf818f18c510d58c0d00e", + "source_lock_sha256": "139f36701e0e47f45bf99fba2cc2fa59b417f2ee801dad3a064455d5b464a459", "target_head_quantization": false, - "vllm_overrides": 14, + "vllm_overrides": 24, "vllm_parent_files": 2905 }, "cuda_initialized": false, @@ -50,18 +50,18 @@ }, "rocenante_lazy_import": "/opt/spark-sircl/b12x_overlay/b12x/comm/roce/__init__.py", "sircl_native_sha256": "61aa0ec56a1b438439bed8611dab0353d2c72c10af02bbd917fb77c87b33e5fc", - "source_receipt_sha256": "09f3d010db1e15d8c5bb414cd75a7aebbcdf4e6a73d2224e65933cfc3348e3c4", + "source_receipt_sha256": "45b935dd4075c7c4e208c5d3568771860bfad5dcbb07a954ac664d03e19f6eeb", "sparkcache_commit": "66057174301a4759ca3a45207ea41016689449cb", "status": "research-only", "vllm_commit": "e02b174693e13859de61811b5e8cd13d5308e259", "vllm_native_extensions": 15 }, - "limitation": "No full-model, GPU, fabric, or four-rank serving test was performed for this image.", + "limitation": "This receipt records build/content checks. The linked functional record identifies exact-image GPU, serving, restart and cache checks.", "parent_image_id": "sha256:5e32aaa1bbe3559e81db7706ed4286248f18d27cfdb186f6b851bf786eb43075", "parent_layers_retained": 81, "platform": "linux/arm64", "schema": "sparkring-mtp3-mesh-image-receipt/v1", - "source_receipt_sha256": "09f3d010db1e15d8c5bb414cd75a7aebbcdf4e6a73d2224e65933cfc3348e3c4", + "source_receipt_sha256": "45b935dd4075c7c4e208c5d3568771860bfad5dcbb07a954ac664d03e19f6eeb", "status": "research-only", "verification_command": [ "docker", @@ -86,9 +86,10 @@ "PYTHONDONTWRITEBYTECODE=1", "--entrypoint", "python3", - "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", + "sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f", "-I", "/opt/sparkring/bin/verify-mtp3-mesh-image.py", "--inside-image" - ] + ], + "functional_record": "../../performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md" } diff --git a/runtime/glm53-spark-mtp3-mesh/managed_install.py b/runtime/glm53-spark-mtp3-mesh/managed_install.py index f505a743..634ff773 100644 --- a/runtime/glm53-spark-mtp3-mesh/managed_install.py +++ b/runtime/glm53-spark-mtp3-mesh/managed_install.py @@ -29,6 +29,7 @@ 'runtime/glm53-spark-mtp3-mesh/profile.py', 'runtime/glm53-spark-mtp3-mesh/inspect_fabric.py', 'runtime/glm53-spark-mtp3-mesh/pins.json', + 'runtime/glm53-spark-mtp3-mesh/compute/source-lock.json', 'runtime/glm53-flash-jj-r8-gb10/pins.json', 'runtime/glm53-flash-jj-r8-gb10/warmup_dflash.py', 'runtime/glm53-flash-jj-r8-gb10/launch-rank.sh', diff --git a/runtime/glm53-spark-mtp3-mesh/pins.json b/runtime/glm53-spark-mtp3-mesh/pins.json index 2aec2f66..3267552b 100644 --- a/runtime/glm53-spark-mtp3-mesh/pins.json +++ b/runtime/glm53-spark-mtp3-mesh/pins.json @@ -4,7 +4,7 @@ "image_pins": "../glm53-flash-jj-r8-gb10/pins.json", "compute": { "source_lock": "compute/source-lock.json", - "source_lock_sha256": "9a9568bc9f6bc34f4ee2ac5ebe881c3b8a6de289b26aabdc4056ea2852e49fd9", + "source_lock_sha256": "139f36701e0e47f45bf99fba2cc2fa59b417f2ee801dad3a064455d5b464a459", "vllm_base_revision": "e02b174693e13859de61811b5e8cd13d5308e259", "b12x_revision": "ef308bac0f3b3eb8fea63e4013afc0c2ea1c6301", "b12x_tree": "dcf039e5e754136275835ea997e6b9abbb6b15ae", @@ -66,7 +66,7 @@ "cache_identity": { "draft_policy": "separate", "draft_checkpoint_source": "target.checkpoint_identity", - "namespace": "glm53-spark-df116c4f-mtp3-nvfp4-a16-b58f34ea-mesh4204fabc-tail-cow-v2", - "compatibility": "The namespace separates the NVFP4/BF16 proposal head and B12X b58f34ea computation from cache entries produced with a shared BF16 head. The target checkpoint remains df116c4f; persistent restore requires this profile's matching compute and cache geometry." + "namespace": "glm53-spark-df116c4f-mtp3-nvfp4-a16-c139f3670-mesh69313e19-tail-cow-v2", + "compatibility": "The namespace binds the NVFP4/BF16 proposal head, compute source lock139f3670, and mesh bundle69313e19. Do not relabel cache entries from another compute composition. Persistent restore requires matching checkpoint, computation, and cache geometry." } } diff --git a/runtime/glm53-spark-mtp3-mesh/public-image.json b/runtime/glm53-spark-mtp3-mesh/public-image.json index 183fccad..7f19c99a 100644 --- a/runtime/glm53-spark-mtp3-mesh/public-image.json +++ b/runtime/glm53-spark-mtp3-mesh/public-image.json @@ -5,17 +5,17 @@ "anonymous_manifest_read": true, "anonymous_config_read": true, "anonymous_pull": true, - "anonymous_pull_method": "Docker pull on the build host with an empty client credential directory; local image layers already present", - "public_reference": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:b65d427f9be49c97d57e404ad1a6118769c1119df876a8944c1f186a6b380c5d", - "tag": "glm53-spark-mtp3-stream69313e19", - "manifest_digest": "sha256:b65d427f9be49c97d57e404ad1a6118769c1119df876a8944c1f186a6b380c5d", - "config_image_id": "sha256:69c794bf0704e89aa8e2364fb65b972618cf55a665cd3a8ff80a76a1d3280766", + "anonymous_pull_method": "Empty Docker client credential directory; local image layers already present", + "public_reference": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:67dc0ae453baaae6831ccec1d259b4ef8b236a8b0dc9f747d901b95c66ec1987", + "tag": "glm53-spark-mtp3-c139f3670-mesh69313e19", + "manifest_digest": "sha256:67dc0ae453baaae6831ccec1d259b4ef8b236a8b0dc9f747d901b95c66ec1987", + "config_image_id": "sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f", "platform": "linux/arm64", "layer_count": 89, "all_layer_diff_ids_match_tested_image": true, - "layer_identity_reference": "Built image verified by image-receipt.json; serving-image source parity is recorded separately", + "layer_identity_reference": "The tested config-image ID is identical to the published config-image ID.", "package_url": "https://github.com/FujitsuPolycom/sparkring/pkgs/container/sparkring-glm53-sparkcache", "content_receipt": "image-receipt.json", "compute_equivalence": "compute-image-equivalence.json", - "functional_record": "../../performance/records/glm53-flash/spark-mtp3-nvfp4-proposal-head-20260905.md" + "functional_record": "../../performance/records/glm53-flash/spark-mtp3-compute-stream-safety-20260906.md" } diff --git a/runtime/glm53-spark-mtp3-mesh/qualification/run_native.py b/runtime/glm53-spark-mtp3-mesh/qualification/run_native.py index e7f945c0..96745d03 100644 --- a/runtime/glm53-spark-mtp3-mesh/qualification/run_native.py +++ b/runtime/glm53-spark-mtp3-mesh/qualification/run_native.py @@ -85,9 +85,13 @@ def main() -> None: parser.add_argument("--output", type=Path, required=True) parser.add_argument("--rows", nargs="+", type=int, default=[4, 20, 28, 64]) parser.add_argument("--port", type=int, default=29960) + parser.add_argument("--mode", choices=("correctness", "streams"), default="correctness") parser.add_argument("--execute-authorized", action="store_true") args = parser.parse_args() plan = make_plan(args.launch, args.image_receipt, args.rows, args.port) + source_path = HERE / ("stream_roce.py" if args.mode == "streams" else "native_roce.py") + plan["mode"] = args.mode + plan["source_sha256"] = profile.sha(source_path) if not args.execute_authorized: print(json.dumps(plan, indent=2)) return @@ -100,7 +104,7 @@ def main() -> None: (args.output / f"preflight-r{rank['rank']}.json").write_text(json.dumps({"containers": state, "image": image}, indent=2)) if state["returncode"] or state["stdout"].strip() or image["returncode"] or image["stdout"].strip() != plan["image"]: raise RuntimeError("Require no running containers and the exact image on every rank") - source = (HERE / "native_roce.py").read_text() + source = source_path.read_text() for cell in plan["cells"]: with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: results = list(pool.map(lambda rank: remote(rank["host"], rank["argv"], source), cell["ranks"])) diff --git a/runtime/glm53-spark-mtp3-mesh/qualification/stream_roce.py b/runtime/glm53-spark-mtp3-mesh/qualification/stream_roce.py new file mode 100644 index 00000000..ef8960ba --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/qualification/stream_roce.py @@ -0,0 +1,81 @@ +"""Validate bundled RoCEnante staging across CUDA streams with exact BF16 sums.""" +import argparse +import json +import torch +import torch.distributed as dist + +parser=argparse.ArgumentParser() +parser.add_argument('--bytes',type=int,required=True) +args,_=parser.parse_known_args() +dist.init_process_group('gloo') +rank=dist.get_rank() +assert dist.get_world_size()==4 +torch.cuda.set_device(0) +import b12x.comm # noqa: E402 -- device selection precedes transport import +b12x.comm.__path__.insert(0,'/opt/spark-sircl/b12x_overlay/b12x/comm') +from b12x.comm import roce # noqa: E402 -- resolve only the image-bundled transport +runtime=roce.AllReduce.from_exchange_group(exchange_group=dist.group.WORLD,device=torch.device('cuda',0), + max_size=2<<20,max_gather_bytes=2<<20) +runtime.prepare((torch.bfloat16,)) +numel=args.bytes//2 +streams=[torch.cuda.Stream(),torch.cuda.Stream()] +inputs=[torch.empty(numel+1,device='cuda',dtype=torch.bfloat16)[1:] for _ in range(16)] +outputs=[torch.empty_like(inp.new_empty(numel+1))[1:] for inp in inputs] +assert all(t.data_ptr()%16 for t in inputs+outputs) +# Establish shared scratch before exercising asynchronous alternating callers. +inputs[0].fill_(rank+1) +runtime.all_reduce(inputs[0],out=outputs[0]) +torch.cuda.synchronize() +dist.barrier() +for i,(inp,out) in enumerate(zip(inputs,outputs)): + with torch.cuda.stream(streams[i%2]): + inp.fill_(rank+1+i) + out.fill_(-123) + runtime.all_reduce(inp,out=out) +torch.cuda.synchronize() +runtime.check_health() +assert all(torch.equal(out,torch.full_like(out,10+4*i)) for i,out in enumerate(outputs)) +dist.barrier() +graph=torch.cuda.CUDAGraph() +streams[0].wait_stream(torch.cuda.current_stream()) +with torch.cuda.graph(graph,stream=streams[0]): + runtime.all_reduce(inputs[0],out=outputs[0]) +for i in (20,21): + inputs[0].fill_(rank+1+i) + outputs[0].fill_(-321) + graph.replay() + torch.cuda.synchronize() + runtime.check_health() + assert torch.equal(outputs[0],torch.full_like(outputs[0],10+4*i)) +dist.barrier() +# A separate Python capture context must not admit another stream in one CUDA capture. +probe=torch.zeros(1,device='cuda') +probe.add_(1) +torch.cuda.synchronize() +guard_graph=torch.cuda.CUDAGraph() +rejected=False +with torch.cuda.graph(guard_graph,stream=streams[0]): + probe.add_(1) + with runtime.capture(): + runtime._order_stream(True) + streams[1].wait_stream(streams[0]) + with torch.cuda.stream(streams[1]): + with runtime.capture(): + try: + runtime._order_stream(True) + except RuntimeError as error: + assert 'one stream' in str(error) + rejected=True + streams[0].wait_stream(streams[1]) +assert rejected +torch.cuda.synchronize() +runtime.check_health() +record={'rank':rank,'payload_bytes':args.bytes,'alternating_stream_calls':16, + 'misaligned_input_output':True,'changed_input_graph_replays':2, + 'capture_stream_rejected':rejected,'passed':True} +rows=[None]*4 +dist.all_gather_object(rows,record) +if rank==0: + print('EVIDENCE_JSON '+json.dumps({'checks':rows,'passed':all(row['passed'] for row in rows)}),flush=True) +runtime.close() +dist.destroy_process_group() From a5097d363c56d0cdea6182e8da4e0ce8b62b4e68 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:09:14 -0500 Subject: [PATCH 08/16] Validate DFlash profile content in the shared GLM table --- scripts/test_glm53_flash_profile.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/test_glm53_flash_profile.py b/scripts/test_glm53_flash_profile.py index bc7867b4..05bdc9a4 100644 --- a/scripts/test_glm53_flash_profile.py +++ b/scripts/test_glm53_flash_profile.py @@ -459,14 +459,16 @@ def test_public_glm53_benchmark_is_sanitized_and_front_page_lists_dcp_profiles() assert "api_key" not in text.lower() readme = README_PATH.read_text(encoding="utf-8") - assert "GLM-5.3 Flash NVFP4 target" in readme + assert "### GLM-5.3 Flash" in readme assert "external BF16 DFlash2" in readme - assert "| DCP1 | 4 Sparks · TP4/DCP1 |" in readme - assert "| DCP2 | 4 Sparks · TP4/DCP2 |" in readme - assert "| **DCP4 preferred** | **4 Sparks · TP4/DCP4** |" in readme - assert "| ~1.30M tokens |" in readme - assert "| ~2.90M tokens |" in readme - assert "| **~4.32M tokens** |" in readme + for dcp, capacity in ((1, "~1.30M tokens"), (2, "~2.90M tokens"), (4, "~4.32M tokens")): + row = next(line for line in readme.splitlines() + if line.startswith(f"| DFlash2/SIRCL · DCP{dcp}")) + assert f"4 Sparks · TP4/DCP{dcp}" in row + assert "FP8 · 24 GiB/rank" in row + assert capacity in row + assert "docs/GLM53_JJ_R8_GB10_SPARKCACHE_TP4_QUICKSTART.md" in row + assert "DCP4 is the preferred DFlash2 profile" in readme assert "26/30/24 GiB" in readme assert "b12x-kda-dcp4-20260903.md" in readme assert "| 16K | 2,649 (16K scout) | 37.97 | — | C4: 90.36 | — |" in readme From 17157a408133ceb2c47126390eea6f5a355442a1 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:27:42 -0500 Subject: [PATCH 09/16] Fix B12X indexer publication barrier and detect output stalls --- docs/ISSUE224_ENGINE_STALL.md | 89 ++ docs/ISSUE224_INDEXER_BARRIER.md | 191 ++++ .../harnesses/indexer_barrier/README.md | 21 + .../indexer_barrier/gpu_barrier_probe.py | 111 ++ .../indexer_barrier/repro_indexer_barrier.py | 135 +++ .../indexer_barrier/stress_indexer.py | 77 ++ .../glm53-flash/issue224-dgx4-evidence.json | 986 ++++++++++++++++++ .../glm53-flash/issue224-dgx4-validation.md | 115 ++ runtime/glm53-flash-jj-r8-gb10/README.md | 31 + runtime/glm53-flash-jj-r8-gb10/build_image.py | 10 + runtime/glm53-flash-jj-r8-gb10/launch-rank.sh | 10 +- .../patch_indexer_barrier.py | 44 + .../runtime.env.example | 4 + .../scheduler_liveness.py | 42 +- .../serve_with_warmup.py | 3 + .../test_image_contract.py | 34 + .../test_launcher_contract.py | 13 + .../test_scheduler_liveness.py | 101 ++ .../test_serve_with_warmup.py | 2 + 19 files changed, 2017 insertions(+), 2 deletions(-) create mode 100644 docs/ISSUE224_ENGINE_STALL.md create mode 100644 docs/ISSUE224_INDEXER_BARRIER.md create mode 100644 performance/harnesses/indexer_barrier/README.md create mode 100644 performance/harnesses/indexer_barrier/gpu_barrier_probe.py create mode 100644 performance/harnesses/indexer_barrier/repro_indexer_barrier.py create mode 100644 performance/harnesses/indexer_barrier/stress_indexer.py create mode 100644 performance/records/glm53-flash/issue224-dgx4-evidence.json create mode 100644 performance/records/glm53-flash/issue224-dgx4-validation.md create mode 100644 runtime/glm53-flash-jj-r8-gb10/patch_indexer_barrier.py diff --git a/docs/ISSUE224_ENGINE_STALL.md b/docs/ISSUE224_ENGINE_STALL.md new file mode 100644 index 00000000..c87565bd --- /dev/null +++ b/docs/ISSUE224_ENGINE_STALL.md @@ -0,0 +1,89 @@ +# Running-request stall: evidence and recovery detection + +The [source-level indexer investigation](ISSUE224_INDEXER_BARRIER.md) identifies +a histogram-publication race and includes a GPU-tested kernel fix and targeted +isolation experiment. This document covers the fallback liveness detector. + +Status: implemented for offline output-stall detection. The separate kernel +fix addresses the confirmed publication race behind the suspected deadlock in +[issue #224](https://github.com/FujitsuPolycom/sparkring/issues/224). No full model +soak or deployment of the fix to serving containers is claimed. + +## Source evidence + +The operator pins identify the affected vLLM commit as +`e02b174693e13859de61811b5e8cd13d5308e259` and the earlier comparison commit as +`22ffe1401ca9bd3e4503e62de7b414deca7661a1`, both available in +`https://github.com/FujitsuPolycom/vllm`. + +Comparing those exact commits shows no differences in +`vllm/v1/executor/multiproc_executor.py`, +`vllm/distributed/device_communicators/shm_broadcast.py`, or +`vllm/v1/engine/core.py`. The 15 changed files concern B12X attention/model +paths, associated tests, and argument handling. Timing changes can expose a +pre-existing race, so this does not exonerate the executor. + +The affected queue already caps shared-memory reader waits at five seconds +(`SHM_READER_RECHECK_INTERVAL_MS`). A stack sampled in `poll()` does not show +that the individual poll is indefinite. Lost notification alone should not +permanently hide a published shared-memory slot with this source. + +With async scheduling enabled, `WorkerProc.handle_output()` puts outputs on +`async_output_queue`. A separate `async_output_busy_loop()` invokes +`enqueue_output()`, which calls `AsyncModelRunnerOutput.get_output()` before +publishing the response. The main worker can therefore wait for its next RPC +while its response thread is still blocked. Main-thread-only stacks cannot +establish that a response was produced or lost. + +## Discriminating evidence to collect during a hang + +Capture all Python threads in EngineCore and every TP worker, twice at least +ten seconds apart, before stopping the stack. Use `py-spy dump --pid PID` +inside each relevant process namespace. Include native frames if supported. +Preserve the full output, not just MainThread. + +Check these possibilities in order: + +1. Async output is waiting for CUDA completion. A response thread in + `get_output()` or event synchronization supports this; repeated stacks and + GPU/stream evidence are needed to identify the dependency that cannot finish. +2. Async output failed before publishing. A missing output thread and an + associated traceback support this. Inspect logs from all ranks; the process + itself may still be alive. +3. Transport lost a response or queue state diverged. This requires evidence + that the relevant response was published, plus queue/slot and rank identity. + An idle main thread alone is insufficient. + +Preserve full image digests and hashes of the three source files above from +the running containers. A stale version string or JIT namespace is insufficient +to establish the actual runtime source. Record async-output stacks, metrics, +and logs together so they can be correlated with the same stall. + +## Detection and recovery + +The rank-zero monitor detects sustained running requests without output batches +using `vllm:iteration_tokens_total_count`. The separate 300-second default +`SPARKRING_LIVENESS_OUTPUT_SECONDS` must exceed legitimate prefill and restore +gaps. This standard metric counts output-bearing batches, not every engine +step. It is a fallback heuristic for the single-engine TP4 profile. + +The offline regression feeds fresh unchanged metrics with three running +requests for 300 seconds. Before the change, liveness stays HTTP 200. After +the change, it returns HTTP 503 with `engine_output_stall`. Other checks cover +progress, idle periods, counter reset, recovery, missing metrics, invalid +timeouts, and independent prefill grace. No GPU race is reproduced by these +tests. + +Rebuild the operator wrapper to deploy the monitor; editing a runtime setting +alone cannot update an existing image. This patch does not change published +image pins or qualification receipts. Validate the rebuilt image with both +long healthy prefills/restores and injected output stalls before using its +signal for unattended recovery. Follow the managed deployment's coordinated +stop/recovery procedure; this monitor only reports health. + +Do not automatically replay a timed-out model step. The executor's responses +are ordered without per-call IDs, and a partially completed step can mutate KV +and speculative state. A future executor timeout must invalidate the executor +and propagate failure through the established recovery path, including pending +futures. It needs separate tests for late responses and partial multi-rank +completion; adding a retry to `get_response()` is not a safe fix. diff --git a/docs/ISSUE224_INDEXER_BARRIER.md b/docs/ISSUE224_INDEXER_BARRIER.md new file mode 100644 index 00000000..c19ef1db --- /dev/null +++ b/docs/ISSUE224_INDEXER_BARRIER.md @@ -0,0 +1,191 @@ +# Issue 224: sparse-indexer publication barrier race + +Status: implemented and GPU-tested. The publication race reproduced on all four +GB10 GPUs; the fix prevented it in all 80 probes. The patched indexer passed 15 +GPU correctness tests and 2,000 graph replays with concurrent copies. A full +model soak was not performed. See the +[bounded GPU evidence](../performance/records/glm53-flash/issue224-dgx4-validation.md). + +## Finding + +The strongest source-backed explanation is GPU deadlock in the fused DSA +indexer's cooperative top-k merge. The executor stack is downstream of it. + +Affected source: B12X commit +`9ae41c5cb9935d740456479954b0089f80bd2ef2`, file +`b12x/attention/dsa_indexer/fused_indexer.py`. + +The defect is in `_fused_group_barrier()` at lines 377–386. Its leader publishes +arrival before synchronizing the rest of its thread block: + +```python +if tx == Int32(0): + red_add_global_release_i32(arrival_ptr, Int32(1)) + spin_wait_global_ge_i32(arrival_ptr, (phase + Int32(1)) * ctas_per_group) +cute.arch.sync_threads() +``` + +`_coop_wide_round()` and `_coop_narrow_round()` publish global histogram bins +from multiple warps and then call this helper. There is no block-wide barrier +between those publications and the leader's arrival. A release fence on thread +0 orders that thread's operations; it cannot make an unscheduled publishing +warp finish. NVIDIA documents this distinction in the +[CUDA synchronization and memory-fence rules](https://docs.nvidia.com/cuda/archive/13.0.0/cuda-c-programming-guide/index.html#memory-fence-functions). + +The barrier after the spin only protects consumers inside the same block. +Another block can already have passed its own barrier and started reading the +global histogram while the delayed publisher is still running. + +## Why this becomes a permanent GPU wait + +The cooperative merge computes a pivot from the histogram independently in each +block. It conditionally stops refining when `bin_count == remaining_k`. +Inconsistent histogram snapshots can therefore change the number of subsequent +barriers each block executes. + +One legal schedule, using three blocks and top-k 512: + +1. Block A contributes 512 candidates in a lower pivot bin. Block B contributes + one higher candidate, but its publishing warp is delayed. Block C has no + candidates. Total candidate count is 513. +2. All three block leaders announce arrival. A reads an incomplete histogram + with 512 lower candidates and concludes that refinement is done. +3. B's delayed publication completes. B now sees one higher candidate, leaving + 511 slots to select from the lower bin's 512 candidates. B needs another + radix round. +4. A and C do not enter that round. B increments the cumulative arrival count + from 3 to 4 and spins waiting for 6. The missing arrivals cannot occur. + +The model collapses matching coarse/fine bins into two abstract bins and models +one leader plus one representative nonleader warp per block. It executes an +AST-lowered copy of the actual barrier helper; CUDA scheduling and code generation +are not executed. This is evidence of a reachable protocol failure, not a +hardware reproduction or a measurement of its production frequency. + +The accompanying standalone model uses only Python's standard library. Run it +against a clean checkout of the affected B12X pin, then apply the candidate patch +and repeat: + +```bash +python repro_indexer_barrier.py /path/to/b12x/b12x/attention/dsa_indexer/fused_indexer.py --expect deadlock +git -C /path/to/b12x apply /path/to/issue224-b12x-barrier.patch +python repro_indexer_barrier.py /path/to/b12x/b12x/attention/dsa_indexer/fused_indexer.py --expect complete +``` + +The evidence JSON records both traces: before the change, arrival is 4 with a +block waiting for 6; after the change, arrival reaches 6 with no pending actors. + +## Full response path + +```text +EngineCore.step_with_batch_queue() + -> MultiprocExecutor.execute_model() / sample_tokens() + -> worker main thread submits model GPU work + -> B12xSparseIndexer.forward() + -> dsa_indexer.run() -> fused paged indexer + -> cooperative histogram merge can deadlock + -> AsyncGPUModelRunnerOutput enqueues output copy + -> copy stream waits for the model stream + -> WorkerProc.handle_output() queues the asynchronous output object + -> worker main thread returns to RPC dequeue + -> async_output_busy_loop() -> enqueue_output() -> get_output() + -> async_copy_ready_event.synchronize() cannot finish + -> response never gets enqueued + -> EngineCore waits in get_response() +``` + +Relevant vLLM locations at `e02b174693e13859de61811b5e8cd13d5308e259`: + +- `vllm/v1/engine/core.py`: `step_with_batch_queue()` schedules ahead, then drains + the oldest result through `future.result()`. +- `vllm/v1/executor/multiproc_executor.py`: `handle_output()` sends asynchronous + results to a separate thread. `enqueue_output()` must resolve `get_output()` + before publishing a response. All main worker threads can be idle during this + failure. +- `vllm/v1/worker/gpu_model_runner.py:326`: the output copy stream waits for the + current model stream. Line 356 synchronizes the output-completion event. + +Other ranks can be waiting in later GPU collectives when one rank stops making +progress. The specific kernel running on each rank still requires a GPU trace. +GPU utilization alone does not identify a kernel or prove this attribution. + +## Regression evidence + +The earlier image's receipt pins B12X to +`6255090a03b12c3f7d552102a02fac0b542fb8c9`, while the affected operator image +pins `9ae41c5cb9935d740456479954b0089f80bd2ef2`. + +B12X commit `357576e6d49a2d9fbf623cd73542826fdf55bb8e` introduces the separate +12/12/8-bit cooperative merge and its conditional refinement. It is not an +ancestor of the earlier pin and is an ancestor of the affected pin. The earlier +merge also has publication-order concerns, but its main refinement loop uses a +fixed four rounds; it does not use this new early-exit protocol. Absence of +observed hangs in the earlier image is not proof that it is race-free. + +The affected GB10 profile's `attention.dsa_indexer` entry contains only +`backend: native`. Missing `fused_merge` resolves to `auto`, which now resolves +to cooperative for multi-block groups. GLM's 32-head/top-k-512 shape permits the +fused path for decode plans up to 16 rows on SM121. With 48 SMs and a 16-row +plan, the scratch planner assigns three blocks per group. Request concurrency +and query-row count are not interchangeable, especially with speculation. + +This makes the failure path reachable under the recorded default composition. +The exact live plan and group shape at each reported hang are not available. + +## Corrections to the original diagnosis + +The exact vLLM comparison from `22ffe140` to `e02b1746` leaves the executor, +shared-memory queue, and EngineCore files unchanged. Their source already has: + +- Five-second shared-memory rechecks, independent of notification delivery. +- `VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS`, default 300, passed to both execution + and sampling RPCs. `collective_rpc()` computes a deadline and passes the + remaining time to `dequeue()`. + +A stack in `poll()` cannot establish an indefinite poll. A supervisor restarting +after 60 seconds can prevent the 300-second timeout from being observed. If the +same RPC is confirmed to remain blocked beyond its configured deadline, inspect +the actual container files, environment, and closure's method/deadline; that is +additional evidence not explained by the source-pinned timeout behavior. + +## Candidate fix and isolation experiment + +Add `cute.arch.sync_threads()` at entry to `_fused_group_barrier()`, before the +leader publishes arrival. Keep the existing trailing barrier. This makes every +publishing warp participate before peers are allowed to consume the histogram. +Bump the fused indexer's `KernelCompileSpec` revision from 1 to 2 so existing +cached compiled artifacts do not reuse the old protocol. + +The kernel patch contains those changes only. The image builder applies +`runtime/glm53-flash-jj-r8-gb10/patch_indexer_barrier.py` to the exported B12X +source before creating its source manifest. The transform checks both input +and output hashes and records its own digest in the build receipt. It rejects +unexpected source rather than applying a speculative replacement. The output +uses LF line endings; the original GPU test file used mixed line endings. +Both have identical Python source after newline normalization. + +Existing published image digests are unchanged. Rebuild with this change to +include the fix; restarting an existing image alone does not install it. + +For a targeted A/B run, pass `B12X_FUSED_INDEXER=0` inside every worker container +before startup. The pinned `dsa_indexer/scratch.py` recognizes this switch and +selects the tiled decode route. It bypasses the suspect kernel while retaining +SparkCache, speculation, and the transport configuration. The SparkRing launcher +patch forwards this variable and validates 0/1; the default remains 1. Existing +installed launchers do not forward it automatically. + +Use a separate `JIT_CACHE_NAMESPACE` for that run so captured/compiled model +artifacts cannot hide the changed dispatch. Apply the same setting to all ranks +through the managed coordinated restart procedure. Changing the environment in +an already-running container cannot change its existing plans and graphs. + +If a stall recurs after installing the fix, retain full worker-thread dumps, +image/source hashes, and the selected indexer route. Capture every worker thread +twice, ten seconds apart, and include EngineCore, executor-timeout errors, and +SparkCache metrics. GPU kernel coverage does not establish that every possible +EngineCore stall has the same cause. + +The regular indexer launch also assumes co-resident blocks without requesting +cooperative launch. That is a separate forward-progress risk requiring occupancy +and concurrent-stream validation; the candidate patch does not claim to solve +all possible kernel hangs. diff --git a/performance/harnesses/indexer_barrier/README.md b/performance/harnesses/indexer_barrier/README.md new file mode 100644 index 00000000..ff57058e --- /dev/null +++ b/performance/harnesses/indexer_barrier/README.md @@ -0,0 +1,21 @@ +# Fused indexer barrier checks + +`gpu_barrier_probe.py` uses the installed B12X helper with a delayed publishing +warp. It runs one barrier to expose incomplete publication without intentionally +launching the divergent second round. Run against the unpatched B12X source to +compare the original helper with the additional entry synchronization. + +`stress_indexer.py` checks the patched full indexer against its numerical oracle +over 2,000 CUDA graph replays with concurrent copies. It imports the existing +test helpers from `tests/attention/test_fused_indexer.py` in B12X commit +`9ae41c5cb9935d740456479954b0089f80bd2ef2`. Put that directory on `PYTHONPATH` and +install pytest 8.4.2. Use an isolated container and compilation cache. + +```bash +python3 gpu_barrier_probe.py +PYTHONPATH=/path/to/b12x/tests/attention python3 stress_indexer.py +``` + +See [the GPU evidence](../../records/glm53-flash/issue224-dgx4-validation.md) +for tested image/source hashes and limits. These are kernel checks, not a full +model or SparkCache serving soak. diff --git a/performance/harnesses/indexer_barrier/gpu_barrier_probe.py b/performance/harnesses/indexer_barrier/gpu_barrier_probe.py new file mode 100644 index 00000000..f951e7ef --- /dev/null +++ b/performance/harnesses/indexer_barrier/gpu_barrier_probe.py @@ -0,0 +1,111 @@ +"""Bounded GB10 publication probe using the installed B12X barrier helper. + +One group barrier is executed; no divergent second round is launched. A delayed +publishing warp creates the adversarial schedule without intentionally wedging +the device. The fixed variant inserts the proposed entry synchronization. +""" +import hashlib +import inspect +import json + +import torch +import cutlass +import cutlass.cute as cute +import cuda.bindings.driver as cuda +from cutlass import Int32, Int64 +from cutlass.cute.runtime import from_dlpack +from cutlass.cutlass_dsl import dsl_user_op +from cutlass._mlir.dialects import llvm +import b12x.attention.dsa_indexer.fused_indexer as indexer +from b12x._lib.intrinsics import ( + get_ptr_as_int64, red_add_global_i32, ld_global_acquire_i32, +) + + +@dsl_user_op +def bounded_delay(cycles: Int64, *, loc=None, ip=None): + llvm.inline_asm( + None, [Int64(cycles).ir_value(loc=loc, ip=ip)], + "{ .reg .u64 start, now, elapsed; .reg .pred p; " + "mov.u64 start, %clock64; delay_loop: nanosleep.u32 1000; " + "mov.u64 now, %clock64; sub.u64 elapsed, now, start; " + "setp.lt.u64 p, elapsed, $0; @p bra delay_loop; }", + "l", has_side_effects=True, is_align_stack=False, + asm_dialect=llvm.AsmDialect.AD_ATT, loc=loc, ip=ip, + ) + + +class Probe: + def __init__(self, fixed): + self.fixed = fixed + + @cute.jit + def __call__(self, state: cute.Tensor, output: cute.Tensor, stream: cuda.CUstream): + self.kernel(state, output).launch( + grid=(3, 1, 1), block=(1024, 1, 1), cooperative=True, stream=stream, + ) + + @cute.kernel + def kernel(self, state: cute.Tensor, output: cute.Tensor): + block, _, _ = cute.arch.block_idx() + tx, _, _ = cute.arch.thread_idx() + if block == Int32(0): + if tx == Int32(128): + red_add_global_i32(get_ptr_as_int64(state, Int32(0)), Int32(512)) + if block == Int32(1): + if tx == Int32(128): + bounded_delay(Int64(20000000)) + red_add_global_i32(get_ptr_as_int64(state, Int32(1)), Int32(1)) + if cutlass.const_expr(self.fixed): + cute.arch.sync_threads() + indexer._fused_group_barrier(state, Int32(0), Int32(0), Int32(3), tx) + if tx == Int32(0): + low = ld_global_acquire_i32(get_ptr_as_int64(state, Int32(0))) + high = ld_global_acquire_i32(get_ptr_as_int64(state, Int32(1))) + output[block] = low + high + + +def main(): + source = inspect.getsourcefile(indexer) + props = torch.cuda.get_device_properties(0) + print(json.dumps({"gpu": props.name, "sm": [props.major, props.minor], + "torch": torch.__version__, "source": source, + "source_sha256": hashlib.sha256(open(source, "rb").read()).hexdigest()}, + sort_keys=True), flush=True) + state = torch.zeros(indexer._COOP_STATE_WORDS, device="cuda", dtype=torch.int32) + output = torch.zeros(3, device="cuda", dtype=torch.int32) + stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + state_arg, output_arg = from_dlpack(state), from_dlpack(output) + results = [] + for fixed in (False, True): + fn = cute.compile(Probe(fixed), state_arg, output_arg, stream) + samples = [] + for _ in range(10): + state.zero_() + output.zero_() + fn(state_arg, output_arg, stream) + torch.cuda.synchronize() + samples.append(output.cpu().tolist()) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + state.zero_() + output.zero_() + capture_stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + fn(state_arg, output_arg, capture_stream) + graph_samples = [] + for _ in range(10): + graph.replay() + torch.cuda.synchronize() + graph_samples.append(output.cpu().tolist()) + result = {"entry_sync": fixed, "eager": samples, "graph": graph_samples, + "incomplete_reads": sum(any(x != 513 for x in row) + for row in samples + graph_samples)} + print(json.dumps(result), flush=True) + results.append(result) + assert results[0]["incomplete_reads"] > 0, "baseline did not expose the race" + assert results[1]["incomplete_reads"] == 0, "entry barrier did not fix publication" + print("PASS: actual B12X helper exposes premature publication; entry sync prevents it", flush=True) + + +if __name__ == "__main__": + main() diff --git a/performance/harnesses/indexer_barrier/repro_indexer_barrier.py b/performance/harnesses/indexer_barrier/repro_indexer_barrier.py new file mode 100644 index 00000000..58e0fc43 --- /dev/null +++ b/performance/harnesses/indexer_barrier/repro_indexer_barrier.py @@ -0,0 +1,135 @@ +"""CPU interleaving model of the actual B12X group-barrier helper. + +This does not execute CUDA. It extracts the helper from the given source and +models three blocks, each with a leader and a histogram-publishing warp. +The adversarial schedule delays block B's publisher until block A can scan. +""" + +import argparse +import ast +import json +from pathlib import Path + + +class LowerBarrier(ast.NodeTransformer): + def visit_Expr(self, node): + if not isinstance(node.value, ast.Call): + return node + call = node.value + name = ast.unparse(call.func) + if name == "cute.arch.sync_threads": + args = [ast.Constant("sync")] + elif name == "red_add_global_release_i32": + args = [ast.Constant("arrival")] + elif name == "spin_wait_global_ge_i32": + args = [ast.Constant("wait"), call.args[1]] + else: + return node + return ast.copy_location(ast.Expr(ast.Yield(ast.Tuple(args, ast.Load()))), node) + + +def load_barrier(path): + tree = ast.parse(path.read_text(encoding="utf-8")) + function = next(n for n in tree.body if isinstance(n, ast.FunctionDef) + and n.name == "_fused_group_barrier") + function.decorator_list = [] + function.returns = None + for arg in function.args.args: + arg.annotation = None + function = LowerBarrier().visit(function) + namespace = {"Int32": int, "_fused_state_ptr": lambda *args: 0, + "_FUSED_STATE_ARRIVAL": 0} + exec(compile(ast.fix_missing_locations(ast.Module([function], [])), + str(path), "exec"), namespace) + return namespace["_fused_group_barrier"] + + +def simulate(barrier): + # top-k=512, 513 candidates. A publishes 512 candidates in an abstract lower + # bin, B publishes one in a higher bin, C publishes none. The two-bin model + # collapses the matching coarse/fine histogram updates into one operation. + histogram = {2: 0, 3: 0} + arrival = 0 + decisions = {} + trace = [] + sync_epoch = {block: 0 for block in "ABC"} + sync_arrivals = {} + waiting_sync = {} + + def actor(block, tx): + if tx == 32: + yield ("publish", block) + phase = yield from barrier(None, 0, 0, 3, tx) + if tx == 0: + yield ("scan", block) + yield ("decision", block) + if decisions[block] == "another_round": + yield from barrier(None, 0, phase, 3, tx) + + generators = {f"{b}{t}": actor(b, t) for b in "ABC" for t in (0, 32)} + pending = {key: next(gen) for key, gen in generators.items()} + + def advance(key): + try: + pending[key] = next(generators[key]) + except StopIteration: + pending.pop(key) + + # B32 represents a different warp and has lowest priority: hardware may delay it + # warp while block leaders publish arrival and other blocks consume it. + while pending: + progressed = False + for key in ("A0", "A32", "B0", "C0", "C32", "B32"): + if key not in pending: + continue + op = pending[key] + block = key[0] + if op[0] == "wait" and arrival < op[1]: + continue + if op[0] == "decision" and block not in decisions: + continue + if op[0] == "sync": + if key in waiting_sync: + continue + epoch = sync_epoch[block] + waiting_sync[key] = epoch + reached = sync_arrivals.setdefault((block, epoch), set()) + reached.add(key) + if len(reached) == 2: + sync_epoch[block] += 1 + for peer in sorted(reached): + waiting_sync.pop(peer) + advance(peer) + else: + if op[0] == "publish": + histogram[2 if block == "A" else 3] += {"A": 512, "B": 1, "C": 0}[block] + trace.append(f"{key}: publish -> {histogram.copy()}") + elif op[0] == "arrival": + arrival += 1 + trace.append(f"{key}: arrival={arrival}") + elif op[0] == "scan": + # The pinned kernel stops refining when pivot-bin count + # equals the remaining top-k slots. + count_above = histogram[3] + remaining = 512 - count_above + decisions[block] = ( + "done" if histogram[2] == remaining else "another_round" + ) + trace.append(f"{key}: scan {histogram.copy()} -> {decisions[block]}") + advance(key) + progressed = True + break + if not progressed: + break + return {"deadlocked": bool(pending), "decisions": decisions, + "arrival": arrival, "pending": pending, "trace": trace} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("--expect", choices=("deadlock", "complete"), required=True) + args = parser.parse_args() + result = simulate(load_barrier(args.source)) + print(json.dumps(result, indent=2)) + assert result["deadlocked"] == (args.expect == "deadlock"), result diff --git a/performance/harnesses/indexer_barrier/stress_indexer.py b/performance/harnesses/indexer_barrier/stress_indexer.py new file mode 100644 index 00000000..4a12c9e2 --- /dev/null +++ b/performance/harnesses/indexer_barrier/stress_indexer.py @@ -0,0 +1,77 @@ +"""GLM-shaped patched fused-indexer graph correctness stress on one GB10.""" +import hashlib +import inspect +import json +import time + +import torch +from test_fused_indexer import _build_case, _golden_topk +from b12x.attention.dsa_indexer.fused_indexer import ( + run_fused_paged_indexer, fused_indexer_scratch_capacity, +) + + +def check(idx, val, gold_values, gold_sets): + assert torch.allclose(torch.sort(val, dim=1, descending=True).values, + gold_values, atol=1e-2, rtol=0) + for row, expected in enumerate(gold_sets): + assert set(idx[row].tolist()) == expected, row + + +def main(): + source = inspect.getsourcefile(run_fused_paged_indexer) + print(json.dumps({"source_sha256": hashlib.sha256(open(source, "rb").read()).hexdigest(), + "gpu": torch.cuda.get_device_name(), "torch": torch.__version__}), flush=True) + for rows, max_len in ((3, 65536), (4, 200000), (8, 65536), (16, 65536)): + q, w, k, scales, pages, lengths = _build_case( + rows, 32, max_len, 512, seed=224 + rows, device=torch.device("cuda")) + capacity, state_words = fused_indexer_scratch_capacity(rows, 512, 48) + pack_v = torch.empty(capacity, dtype=torch.float32, device="cuda") + pack_i = torch.empty(capacity, dtype=torch.int32, device="cuda") + state = torch.zeros(state_words, dtype=torch.int32, device="cuda") + idx = torch.empty((rows, 512), dtype=torch.int32, device="cuda") + val = torch.empty((rows, 512), dtype=torch.float32, device="cuda") + kwargs = dict(q_bytes=q.view(torch.uint8), weights=w, + k_quant_bytes=k.view(torch.uint8), k_scales=scales, + real_page_table=pages, seqlens=lengths, num_heads=32, topk=512, + out_indices=idx, out_values=val, merge_threshold=0, + pack_values=pack_v, pack_indices=pack_i, merge_state=state, + merge_state_preinitialized=True) + gold = {} + for length in (4097, max_len): + lengths.fill_(length) + gold[length] = _golden_topk(q, w, k, scales, pages, lengths, 512) + lengths.fill_(max_len) + run_fused_paged_indexer(**kwargs) + torch.cuda.synchronize() + check(idx, val, *gold[max_len]) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run_fused_paged_indexer(**kwargs) + copy_src = torch.ones(16 * 1024 * 1024, dtype=torch.float32, device="cuda") + copy_dst = torch.empty_like(copy_src) + copy_stream = torch.cuda.Stream() + copy_stream.wait_stream(torch.cuda.current_stream()) + started = time.monotonic() + for iteration in range(500): + length = (4097, max_len)[iteration % 2] + lengths.fill_(length) + with torch.cuda.stream(copy_stream): + copy_dst.copy_(copy_src) + graph.replay() + torch.cuda.synchronize() + check(idx, val, *gold[length]) + assert int(state.abs().sum()) == 0 + print(json.dumps({"rows": rows, "heads": 32, "topk": 512, + "lengths": [4097, max_len], "graph_replays": 500, + "concurrent_copy_bytes": copy_src.numel()*4, + "wall_seconds": time.monotonic()-started, + "result": "passed"}), flush=True) + del graph, kwargs, q, w, k, scales, pages, lengths, pack_v, pack_i, state + del idx, val, gold, copy_src, copy_dst, copy_stream + torch.cuda.empty_cache() + print("PASS: 2000 GLM-shaped graph replays with numerical parity and concurrent copies", flush=True) + + +if __name__ == "__main__": + main() diff --git a/performance/records/glm53-flash/issue224-dgx4-evidence.json b/performance/records/glm53-flash/issue224-dgx4-evidence.json new file mode 100644 index 00000000..16ad4aca --- /dev/null +++ b/performance/records/glm53-flash/issue224-dgx4-evidence.json @@ -0,0 +1,986 @@ +{ + "date": "2026-09-06", + "image_id": "sha256:5e32aaa1bbe3559e81db7706ed4286248f18d27cfdb186f6b851bf786eb43075", + "probes": { + "rank0": [ + { + "gpu": "NVIDIA GB10", + "sm": [ + 12, + 1 + ], + "source": "/usr/local/lib/python3.12/dist-packages/b12x/attention/dsa_indexer/fused_indexer.py", + "source_sha256": "d3ec6274e142a4e7d1062ea6d2d99b97db0a02e92bb976c6570ae990b836b18d", + "torch": "2.13.0+cu130" + }, + { + "entry_sync": false, + "eager": [ + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ] + ], + "graph": [ + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ] + ], + "incomplete_reads": 20 + }, + { + "entry_sync": true, + "eager": [ + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ] + ], + "graph": [ + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ] + ], + "incomplete_reads": 0 + } + ], + "rank1": [ + { + "gpu": "NVIDIA GB10", + "sm": [ + 12, + 1 + ], + "source": "/usr/local/lib/python3.12/dist-packages/b12x/attention/dsa_indexer/fused_indexer.py", + "source_sha256": "d3ec6274e142a4e7d1062ea6d2d99b97db0a02e92bb976c6570ae990b836b18d", + "torch": "2.13.0+cu130" + }, + { + "entry_sync": false, + "eager": [ + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ] + ], + "graph": [ + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ] + ], + "incomplete_reads": 20 + }, + { + "entry_sync": true, + "eager": [ + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ] + ], + "graph": [ + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ] + ], + "incomplete_reads": 0 + } + ], + "rank2": [ + { + "gpu": "NVIDIA GB10", + "sm": [ + 12, + 1 + ], + "source": "/usr/local/lib/python3.12/dist-packages/b12x/attention/dsa_indexer/fused_indexer.py", + "source_sha256": "d3ec6274e142a4e7d1062ea6d2d99b97db0a02e92bb976c6570ae990b836b18d", + "torch": "2.13.0+cu130" + }, + { + "entry_sync": false, + "eager": [ + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ] + ], + "graph": [ + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ] + ], + "incomplete_reads": 20 + }, + { + "entry_sync": true, + "eager": [ + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ] + ], + "graph": [ + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ] + ], + "incomplete_reads": 0 + } + ], + "rank3": [ + { + "gpu": "NVIDIA GB10", + "sm": [ + 12, + 1 + ], + "source": "/usr/local/lib/python3.12/dist-packages/b12x/attention/dsa_indexer/fused_indexer.py", + "source_sha256": "d3ec6274e142a4e7d1062ea6d2d99b97db0a02e92bb976c6570ae990b836b18d", + "torch": "2.13.0+cu130" + }, + { + "entry_sync": false, + "eager": [ + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ] + ], + "graph": [ + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ], + [ + 512, + 513, + 512 + ] + ], + "incomplete_reads": 20 + }, + { + "entry_sync": true, + "eager": [ + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ] + ], + "graph": [ + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ], + [ + 513, + 513, + 513 + ] + ], + "incomplete_reads": 0 + } + ] + }, + "full_indexer_pytest": "15 passed, 79 deselected", + "stress": [ + { + "source_sha256": "c63ac2712dc19bc67cb6e892de24751d58a534348e29121623daee64776e2679", + "gpu": "NVIDIA GB10", + "torch": "2.13.0+cu130" + }, + { + "rows": 3, + "heads": 32, + "topk": 512, + "lengths": [ + 4097, + 65536 + ], + "graph_replays": 500, + "concurrent_copy_bytes": 67108864, + "wall_seconds": 0.42420284200125025, + "result": "passed" + }, + { + "rows": 4, + "heads": 32, + "topk": 512, + "lengths": [ + 4097, + 200000 + ], + "graph_replays": 500, + "concurrent_copy_bytes": 67108864, + "wall_seconds": 0.5141781540005468, + "result": "passed" + }, + { + "rows": 8, + "heads": 32, + "topk": 512, + "lengths": [ + 4097, + 65536 + ], + "graph_replays": 500, + "concurrent_copy_bytes": 67108864, + "wall_seconds": 0.5467488969989063, + "result": "passed" + }, + { + "rows": 16, + "heads": 32, + "topk": 512, + "lengths": [ + 4097, + 65536 + ], + "graph_replays": 500, + "concurrent_copy_bytes": 67108864, + "wall_seconds": 0.7687193460005801, + "result": "passed" + } + ], + "sha256": { + "gpu_barrier_probe.py": "3e5cfd24715d1498914915f444abb4431a8a5ba7f5a723bbc1fb736e5540c6ee", + "stress_indexer.py": "c8e5558873bc85ac2bf5d12445b474904ffcbbfcddb5e406a04eca56b73cf3b8", + "test_fused_indexer.py": "ee616af9180585879af8159e781958b739d3e903a19e94ee592df48258b87854", + "fused_indexer.py": "c63ac2712dc19bc67cb6e892de24751d58a534348e29121623daee64776e2679" + }, + "serving_containers_modified": false, + "full_model_soak_run": false +} diff --git a/performance/records/glm53-flash/issue224-dgx4-validation.md b/performance/records/glm53-flash/issue224-dgx4-validation.md new file mode 100644 index 00000000..ff93674c --- /dev/null +++ b/performance/records/glm53-flash/issue224-dgx4-validation.md @@ -0,0 +1,115 @@ +# Issue 224: DGX4 GPU validation + +Status: qualified for the bounded publication-order regression and the tested +single-GPU indexer cases. End-to-end incident resolution remains unqualified. + +## Conditions + +Tests ran on 2026-09-06 on DGX4's four NVIDIA GB10 GPUs (SM121), using isolated +containers from the affected operator image: +`sha256:5e32aaa1bbe3559e81db7706ed4286248f18d27cfdb186f6b851bf786eb43075`. +The runtime reported PyTorch `2.13.0+cu130`; the installed indexer file matched +the affected B12X source byte-for-byte: +`d3ec6274e142a4e7d1062ea6d2d99b97db0a02e92bb976c6570ae990b836b18d`. + +The four existing `sparkring-linked-test-20260905-r*` serving containers were +left running. Their image is newer (`2e41b1e934a8`); inspection of rank 0 +confirmed that its barrier still lacks the entry synchronization. The patch was +not installed in those serving containers. All four remained healthy after the +tests, with zero GPU utilization in the final snapshot. + +## Measurement + +The bounded GPU probe compiles the installed `_fused_group_barrier()` through +CuTe. Three cooperatively launched blocks publish a small histogram; one +nonleader warp delays its contribution by a bounded GPU-clock loop. Each block +records the histogram sum it observes after the barrier. The expected sum is +513. There is only one group barrier, so the test exposes premature consumption +without intentionally executing the divergent second round that would hang. + +Each GPU ran 10 eager and 10 graph-replay probes with the original helper, then +the same 20 probes with the proposed entry synchronization. The first graph +harness attempt incorrectly used the pre-capture stream and was rejected; the +recorded successful run uses the active capture stream and verifies nonzero +results. No result from the rejected attempt is included in the totals. + +The full-indexer tests mounted the patched source read-only into a separate +container on rank 0. Its SHA-256 is +`c63ac2712dc19bc67cb6e892de24751d58a534348e29121623daee64776e2679`. +The patch adds the block synchronization and increases the fused-kernel compile +revision from 1 to 2. Compilation caches were private to the test directory. + +The image-builder transform emits the same source with LF line endings: +`49f6fd916fd1ccf94311ee99427551edbd0dc3a5de23aeeb426418370f76f66d`. +That exact output was compiled again in an isolated container on rank 0; all 15 +selected GPU tests passed again. The original 2,000-replay results below belong +to the mixed-line-ending file identified above. + +Fifteen existing GPU tests covered reference top-k values and selected-index +sets, partial pages, short contexts, repeated cooperative merges, padding, +counter cleanup, and graph replay switching between serial/cooperative merge. + +An additional stress harness ran 500 graph replays for each GLM-shaped case +below. Every replay checked sorted top-k values against the reference with +absolute tolerance 0.01, exact selected-index sets, and cleared merge state. +Lengths alternated between 4,097 and the case's maximum. Each replay also +submitted a 64 MiB device-to-device copy on a separate stream. + +## Result + +| Test | Result | +|---|---| +| Original helper, all four GPUs | 80/80 probes observed incomplete data | +| Entry synchronization, all four GPUs | 80/80 probes observed complete data | +| Existing patched full-indexer GPU tests, rank 0 | 15 passed; 79 unrelated cases deselected | +| 3 rows, 32 heads, top-k 512, maximum 65,536 tokens | 500/500 graph replays passed | +| 4 rows, 32 heads, top-k 512, maximum 200,000 tokens | 500/500 graph replays passed | +| 8 rows, 32 heads, top-k 512, maximum 65,536 tokens | 500/500 graph replays passed | +| 16 rows, 32 heads, top-k 512, maximum 65,536 tokens | 500/500 graph replays passed | + +Original probes consistently returned `[512, 513, 512]`: two blocks passed the +barrier before the delayed contribution. Fixed probes consistently returned +`[513, 513, 513]`, in both eager execution and CUDA graph replay. + +## Conclusion + +The publication-order race is confirmed on all four DGX4 GPUs. The proposed +entry synchronization fixes that measured race. The patched full indexer +compiles and passes the tested numerical and graph-replay cases on GB10, +including 2,000 replays with concurrent device copies. + +This supports testing a patched serving image. It does not establish that all +reported EngineCore stalls have the same cause or that the production incident +is resolved. + +## Limitations and next serving test + +The probe deliberately forces a delayed publisher; it does not measure the +natural failure rate. The stress test exercises one indexer on one GPU, uses +synthetic inputs and device copies, and does not run actual SparkCache restores, +TP4 collectives, the full model, or a multi-hour workload. No performance claim +is made from these tests. + +The next gate is a coordinated four-rank canary using a child image that changes +only the barrier fix and its compile revision, with a separate JIT namespace. +Repeat the issue's concurrency-three-or-higher restore/decode workload, retain +full worker-thread stacks on stalls, and compare against the same unpatched +image/configuration. The running test stack has not been replaced. + +## Reproduction artifacts + +[Raw evidence](issue224-dgx4-evidence.json) contains probe and stress observations +and artifact hashes. The [GPU harnesses](../../harnesses/indexer_barrier/README.md) +and the image builder's +[checked transform](../../../runtime/glm53-flash-jj-r8-gb10/patch_indexer_barrier.py) +are checked in. The existing test file comes from the pinned B12X repository. + +Run `gpu_barrier_probe.py` inside the affected image with GPU access. For the +full-indexer tests, apply the checked transform to that image's B12X source in +an isolated child image or use a read-only file mount. Install pytest +8.4.2 in the test environment, then run: + +```bash +python3 -m pytest test_fused_indexer.py -q -x -k 'paged_matches_reference or partial_last_page or short_context_no_radix or preinitialized_state_graph_replay or cooperative_merge_repeated_launches or cooperative_pack_path' +python3 stress_indexer.py +``` diff --git a/runtime/glm53-flash-jj-r8-gb10/README.md b/runtime/glm53-flash-jj-r8-gb10/README.md index a11ee34b..8076bcaf 100644 --- a/runtime/glm53-flash-jj-r8-gb10/README.md +++ b/runtime/glm53-flash-jj-r8-gb10/README.md @@ -362,6 +362,36 @@ and at least one waiting request for 60 seconds. It also returns 503 when SparkCache reports uncertain capture-page ownership. `GET /metrics` on the same port exports the liveness state and blocked duration. +Status: implemented; the output-stall detector has offline regression coverage +and still needs validation under live TP4 cache-restore traffic. +Running requests with no change in `vllm:iteration_tokens_total_count` for +`SPARKRING_LIVENESS_OUTPUT_SECONDS` (default 300 seconds) return HTTP 503 with +reason `engine_output_stall`. Fresh HTTP scrapes do not reset that timer. +Output progress, counter reset, or an observed idle period starts a new window. +Missing output metrics while requests are running eventually report +`metrics_unavailable`, rather than certifying progress. + +This is an output-progress heuristic, not an engine-step heartbeat. The pinned +vLLM increments the histogram when the API receives output-bearing batches; +prefill and external-cache restores can legitimately leave it unchanged. +Set the timeout above the longest measured prefill, restore, or output gap, +with margin. The monitor sums metrics for the single-engine TP4 deployment; +it does not detect one stalled engine hidden by another progressing engine. +The JSON includes `output_stalled_seconds` and `output_iterations`, and the +monitor's metrics include `sparkring:engine_output_stalled_seconds`. + +An unhealthy result does not restart the cluster. Use the deployment's +coordinated stop/recovery procedure after collecting all worker-thread stacks. +See [the executor-stall investigation](../../docs/ISSUE224_ENGINE_STALL.md). +For isolation of the pinned B12X indexer's histogram-publication race, the +launcher accepts `B12X_FUSED_INDEXER=0` and forwards it to the container. Use a +separate JIT cache namespace and coordinated restart on all ranks. This bypass +can change throughput and has not been qualified on the affected cluster. +The image builder applies the GPU-tested publication barrier and compile-cache +revision through `patch_indexer_barrier.py` before generating the B12X source +manifest. This requires rebuilding the image; published image pins remain +unchanged. See [the source trace and fix](../../docs/ISSUE224_INDEXER_BARRIER.md). + Idle KV retention is warning-only. The default 330-second warning interval is longer than the GLM profile's 300-second shared-prefix lease, so an intentional lease is not treated as a dead scheduler. @@ -385,6 +415,7 @@ on an ARM64 CUDA 13 host before invoking the image builder: | SIRCL Python overlay | This checkout plus `runtime/public-overlay-files.json` | | SIRCL ARM64 native library | This checkout plus `sircl-public-build-receipt.json` and `pins.json` `sircl` hashes | | Short KV-metrics logger transform | [`patch_kv_metrics_logging.py`](patch_kv_metrics_logging.py) and its exact vLLM preimage | +| B12X histogram publication barrier | [`patch_indexer_barrier.py`](patch_indexer_barrier.py), with checked source and result hashes | ```bash cmake -S /source/sparkcache/sparkcache/native \ diff --git a/runtime/glm53-flash-jj-r8-gb10/build_image.py b/runtime/glm53-flash-jj-r8-gb10/build_image.py index e7f0460f..2c99e55f 100644 --- a/runtime/glm53-flash-jj-r8-gb10/build_image.py +++ b/runtime/glm53-flash-jj-r8-gb10/build_image.py @@ -325,6 +325,13 @@ def prepare_context( sparkcache_output = context / "bundle/sources/sparkcache" extract_git_subtree(vllm_source, pins["vllm"]["commit"], "vllm", vllm_output) extract_git_subtree(b12x_source, pins["b12x"]["commit"], "b12x", b12x_output) + run( + ( + sys.executable, + HERE / "patch_indexer_barrier.py", + b12x_output / "attention/dsa_indexer/fused_indexer.py", + ) + ) run( ( sys.executable, @@ -461,6 +468,9 @@ def prepare_context( receipt["inputs"]["source_transform/kv_metrics_logging"] = file_sha256( HERE / "patch_kv_metrics_logging.py" ) + receipt["inputs"]["source_transform/indexer_barrier"] = file_sha256( + HERE / "patch_indexer_barrier.py" + ) receipt_path = context / "bundle/receipts/source-receipt.json" receipt_path.write_text( json.dumps(receipt, indent=2, sort_keys=True) + "\n", diff --git a/runtime/glm53-flash-jj-r8-gb10/launch-rank.sh b/runtime/glm53-flash-jj-r8-gb10/launch-rank.sh index 4052aec6..2090f0a3 100644 --- a/runtime/glm53-flash-jj-r8-gb10/launch-rank.sh +++ b/runtime/glm53-flash-jj-r8-gb10/launch-rank.sh @@ -41,6 +41,7 @@ esac : "${DECODE_CONTEXT_PARALLEL_SIZE:=4}" : "${CP_KV_CACHE_INTERLEAVE_SIZE:=auto}" : "${B12X_MLA_CKV_GATHER:=auto}" +: "${B12X_FUSED_INDEXER:=1}" : "${B12X_MLA_CKV_GATHER_MAX_TOKENS:=524288}" : "${NODE_COUNT:=4}" : "${MAX_MODEL_LEN:=1048576}" @@ -78,6 +79,7 @@ esac : "${SPARKRING_LIVENESS_ENABLED:=1}" : "${SPARKRING_LIVENESS_PORT:=8016}" : "${SPARKRING_LIVENESS_BLOCKED_SECONDS:=60}" +: "${SPARKRING_LIVENESS_OUTPUT_SECONDS:=300}" : "${SPARKRING_IDLE_KV_WARN_SECONDS:=330}" : "${SPARKRING_LIVENESS_STALE_SECONDS:=15}" : "${SPARKRING_LIVENESS_SAMPLE_SECONDS:=10}" @@ -174,7 +176,7 @@ for name in \ SPARKCACHE_ASYNC_CAPTURE_SLOT_COUNT \ NCCL_MIN_NCHANNELS NCCL_MAX_NCHANNELS OMP_NUM_THREADS \ TORCHINDUCTOR_COMPILE_THREADS FASTSAFETENSORS_QUEUE_SIZE \ - SPARKRING_LIVENESS_PORT SPARKRING_LIVENESS_BLOCKED_SECONDS \ + SPARKRING_LIVENESS_PORT SPARKRING_LIVENESS_BLOCKED_SECONDS SPARKRING_LIVENESS_OUTPUT_SECONDS \ SPARKRING_IDLE_KV_WARN_SECONDS SPARKRING_LIVENESS_STALE_SECONDS \ SPARKRING_LIVENESS_SAMPLE_SECONDS do @@ -239,6 +241,10 @@ case "${B12X_MLA_CKV_GATHER}" in 0|1) ;; *) die 'B12X_MLA_CKV_GATHER must be auto, 0, or 1' ;; esac +case "${B12X_FUSED_INDEXER}" in + 0|1) ;; + *) die 'B12X_FUSED_INDEXER must be 0 or 1' ;; +esac [[ "${rank}" =~ ^[0-9]+$ ]] || die 'rank must be an unsigned integer' (( rank < NODE_COUNT )) || die "rank must be between 0 and $((NODE_COUNT - 1))" @@ -870,6 +876,7 @@ container_command=(docker "${container_action[@]}" \ -e "SPARKRING_LIVENESS_ENABLED=${SPARKRING_LIVENESS_ENABLED}" \ -e "SPARKRING_LIVENESS_PORT=${SPARKRING_LIVENESS_PORT}" \ -e "SPARKRING_LIVENESS_BLOCKED_SECONDS=${SPARKRING_LIVENESS_BLOCKED_SECONDS}" \ + -e "SPARKRING_LIVENESS_OUTPUT_SECONDS=${SPARKRING_LIVENESS_OUTPUT_SECONDS}" \ -e "SPARKRING_IDLE_KV_WARN_SECONDS=${SPARKRING_IDLE_KV_WARN_SECONDS}" \ -e "SPARKRING_LIVENESS_STALE_SECONDS=${SPARKRING_LIVENESS_STALE_SECONDS}" \ -e "SPARKRING_LIVENESS_SAMPLE_SECONDS=${SPARKRING_LIVENESS_SAMPLE_SECONDS}" \ @@ -878,6 +885,7 @@ container_command=(docker "${container_action[@]}" \ -e VLLM_GLM53_SPLIT_TARGET_BLOCK_SIZE=512 \ -e VLLM_GLM53_SPLIT_MAMBA_BLOCK_SIZE=512 \ -e "VLLM_B12X_MLA_CKV_GATHER=${B12X_MLA_CKV_GATHER}" \ + -e "B12X_FUSED_INDEXER=${B12X_FUSED_INDEXER}" \ -e "VLLM_B12X_MLA_CKV_GATHER_MAX_TOKENS=${B12X_MLA_CKV_GATHER_MAX_TOKENS}" \ -e "VLLM_CACHE_ROOT=/cache/jit/vllm/${JIT_CACHE_NAMESPACE}" \ -e "B12X_CUTE_COMPILE_CACHE_DIR=/cache/jit/b12x/${JIT_CACHE_NAMESPACE}" \ diff --git a/runtime/glm53-flash-jj-r8-gb10/patch_indexer_barrier.py b/runtime/glm53-flash-jj-r8-gb10/patch_indexer_barrier.py new file mode 100644 index 00000000..20fe3775 --- /dev/null +++ b/runtime/glm53-flash-jj-r8-gb10/patch_indexer_barrier.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Apply the GB10-tested histogram publication barrier to pinned B12X source.""" +from __future__ import annotations + +import argparse +import hashlib +from pathlib import Path + + +BEFORE_SHA256 = "d3ec6274e142a4e7d1062ea6d2d99b97db0a02e92bb976c6570ae990b836b18d" +AFTER_SHA256 = "49f6fd916fd1ccf94311ee99427551edbd0dc3a5de23aeeb426418370f76f66d" +_BEFORE = ''' """Grid barrier over the group's CTAs on the arrival counter; returns the next phase.""" + arrival_ptr = _fused_state_ptr(state, group_id, Int32(_FUSED_STATE_ARRIVAL)) +''' +_AFTER = ''' """Grid barrier over the group's CTAs on the arrival counter; returns the next phase.""" + # Every publishing warp must finish before the leader releases this CTA's + # arrival; otherwise peers can scan partial histograms and diverge in rounds. + cute.arch.sync_threads() + arrival_ptr = _fused_state_ptr(state, group_id, Int32(_FUSED_STATE_ARRIVAL)) +''' +_OLD_CACHE = '"attention.indexer.fused_indexer", 1, cache_key, labels=labels' +_NEW_CACHE = '"attention.indexer.fused_indexer", 2, cache_key, labels=labels' + + +def apply_patch(path: Path) -> None: + source = path.read_bytes() + digest = hashlib.sha256(source).hexdigest() + if digest == AFTER_SHA256: + return + if digest != BEFORE_SHA256: + raise RuntimeError(f"unsupported B12X fused indexer source: {digest}") + text = source.decode("utf-8") + if text.count(_BEFORE) != 1 or text.count(_OLD_CACHE) != 1: + raise RuntimeError("pinned B12X barrier or compile revision differs") + patched = text.replace(_BEFORE, _AFTER, 1).replace(_OLD_CACHE, _NEW_CACHE, 1).encode("utf-8") + if hashlib.sha256(patched).hexdigest() != AFTER_SHA256: + raise RuntimeError("B12X barrier transform differs from the GPU-tested source") + path.write_bytes(patched) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("path", type=Path) + apply_patch(parser.parse_args().path) diff --git a/runtime/glm53-flash-jj-r8-gb10/runtime.env.example b/runtime/glm53-flash-jj-r8-gb10/runtime.env.example index 79e99e2d..0931cda2 100644 --- a/runtime/glm53-flash-jj-r8-gb10/runtime.env.example +++ b/runtime/glm53-flash-jj-r8-gb10/runtime.env.example @@ -101,6 +101,10 @@ DFLASH_WARMUP_TIMEOUT_SECONDS=600 SPARKRING_LIVENESS_ENABLED=1 SPARKRING_LIVENESS_PORT=8016 SPARKRING_LIVENESS_BLOCKED_SECONDS=60 +# Running requests without output progress. Allow for the longest prefill/restore. +SPARKRING_LIVENESS_OUTPUT_SECONDS=300 +# Set to 0 to isolate fused sparse-indexer hangs. May reduce decode throughput. +B12X_FUSED_INDEXER=1 # Keep this above the 300-second shared-prefix lease so normal retention warns # only after the lease has had time to release its GPU pages. SPARKRING_IDLE_KV_WARN_SECONDS=330 diff --git a/runtime/glm53-flash-jj-r8-gb10/scheduler_liveness.py b/runtime/glm53-flash-jj-r8-gb10/scheduler_liveness.py index ed625fd4..742399f3 100644 --- a/runtime/glm53-flash-jj-r8-gb10/scheduler_liveness.py +++ b/runtime/glm53-flash-jj-r8-gb10/scheduler_liveness.py @@ -4,6 +4,7 @@ from __future__ import annotations import json +import math import re import threading import time @@ -17,6 +18,7 @@ "waiting": "vllm:num_requests_waiting", "kv_usage": "vllm:kv_cache_usage_perc", "uncertain_ranks": "vllm:sparkcache_capture_ownership_uncertain_ranks", + "output_iterations": "vllm:iteration_tokens_total_count", } @@ -27,6 +29,8 @@ def _metric_sum(text: str, name: str, *, required: bool = True) -> float: values = [float(match.group(1)) for match in pattern.finditer(text)] if not values and required: raise ValueError(f"metrics response does not contain {name}") + if any(not math.isfinite(value) or value < 0 for value in values): + raise ValueError(f"metrics response contains invalid {name}") return sum(values) @@ -39,21 +43,26 @@ def __init__( blocked_timeout_seconds: float, idle_kv_warn_seconds: float, stale_sample_seconds: float, + output_timeout_seconds: float = 300.0, clock: Callable[[], float] = time.monotonic, ) -> None: for name, value in ( ("blocked timeout", blocked_timeout_seconds), ("idle KV warning", idle_kv_warn_seconds), ("stale sample timeout", stale_sample_seconds), + ("output timeout", output_timeout_seconds), ): - if value <= 0: + if not math.isfinite(value) or value <= 0: raise ValueError(f"{name} must be positive") self._blocked_timeout = float(blocked_timeout_seconds) self._idle_kv_warn = float(idle_kv_warn_seconds) self._stale_sample = float(stale_sample_seconds) + self._output_timeout = float(output_timeout_seconds) self._clock = clock self._lock = threading.Lock() self._blocked_since: float | None = None + self._output_stalled_since: float | None = None + self._last_output_iterations: float | None = None self._idle_nonfall_since: float | None = None self._last_idle_kv: float | None = None self._last_success: float | None = None @@ -77,7 +86,23 @@ def observe(self, metrics_text: str) -> None: ), } now = self._clock() + # This histogram counts output-bearing batches received by the API, + # not HTTP scrapes. It can stay flat during a legitimate long prefill. + output_iterations = _metric_sum( + metrics_text, _METRICS["output_iterations"], + required=values["running"] > 0, + ) with self._lock: + if values["running"] > 0: + if ( + self._output_stalled_since is None + or output_iterations != self._last_output_iterations + ): + # A decrease starts a new window after an engine restart. + self._output_stalled_since = now + else: + self._output_stalled_since = None + self._last_output_iterations = output_iterations if values["running"] == 0 and values["waiting"] > 0: if self._blocked_since is None: self._blocked_since = now @@ -106,6 +131,11 @@ def observe_error(self, error: BaseException) -> None: def snapshot(self) -> dict[str, object]: now = self._clock() with self._lock: + output_stalled_seconds = ( + max(0.0, now - self._output_stalled_since) + if self._output_stalled_since is not None + else 0.0 + ) blocked_seconds = ( max(0.0, now - self._blocked_since) if self._blocked_since is not None @@ -132,6 +162,9 @@ def snapshot(self) -> dict[str, object]: elif blocked_seconds >= self._blocked_timeout: healthy = False reason = "scheduler_capacity_stall" + elif output_stalled_seconds >= self._output_timeout: + healthy = False + reason = "engine_output_stall" warnings = [] if idle_kv_seconds >= self._idle_kv_warn: warnings.append("idle_kv_not_falling") @@ -147,6 +180,8 @@ def snapshot(self) -> dict[str, object]: "uncertain_ranks" ], "blocked_seconds": blocked_seconds, + "output_stalled_seconds": output_stalled_seconds, + "output_iterations": self._last_output_iterations, "idle_kv_nonfall_seconds": idle_kv_seconds, "sample_age_seconds": sample_age, "last_sample_error": self._last_error, @@ -160,6 +195,9 @@ def prometheus(self) -> str: values = { "sparkring:scheduler_liveness": int(bool(snapshot["healthy"])), "sparkring:scheduler_blocked_seconds": snapshot["blocked_seconds"], + "sparkring:engine_output_stalled_seconds": snapshot[ + "output_stalled_seconds" + ], "sparkring:idle_kv_nonfall_seconds": snapshot[ "idle_kv_nonfall_seconds" ], @@ -279,11 +317,13 @@ def start_liveness_service( stale_sample_seconds: float, sample_interval_seconds: float, credential: str | None, + output_timeout_seconds: float = 300.0, ) -> SchedulerLivenessService: monitor = SchedulerLiveness( blocked_timeout_seconds=blocked_timeout_seconds, idle_kv_warn_seconds=idle_kv_warn_seconds, stale_sample_seconds=stale_sample_seconds, + output_timeout_seconds=output_timeout_seconds, ) service = SchedulerLivenessService( metrics_url=metrics_url, diff --git a/runtime/glm53-flash-jj-r8-gb10/serve_with_warmup.py b/runtime/glm53-flash-jj-r8-gb10/serve_with_warmup.py index bef2fd3d..3ec746c4 100644 --- a/runtime/glm53-flash-jj-r8-gb10/serve_with_warmup.py +++ b/runtime/glm53-flash-jj-r8-gb10/serve_with_warmup.py @@ -76,6 +76,9 @@ def start_rank_liveness( blocked_timeout_seconds=float( os.environ.get("SPARKRING_LIVENESS_BLOCKED_SECONDS", "60") ), + output_timeout_seconds=float( + os.environ.get("SPARKRING_LIVENESS_OUTPUT_SECONDS", "300") + ), idle_kv_warn_seconds=float( os.environ.get("SPARKRING_IDLE_KV_WARN_SECONDS", "330") ), diff --git a/runtime/glm53-flash-jj-r8-gb10/test_image_contract.py b/runtime/glm53-flash-jj-r8-gb10/test_image_contract.py index 9a443604..0a4a8570 100644 --- a/runtime/glm53-flash-jj-r8-gb10/test_image_contract.py +++ b/runtime/glm53-flash-jj-r8-gb10/test_image_contract.py @@ -5,6 +5,8 @@ import json from pathlib import Path +import pytest + HERE = Path(__file__).resolve().parent ROOT = HERE.parents[1] @@ -25,6 +27,38 @@ def load_module(name: str, path: Path): "jj_r8_metrics_patch", HERE / "patch_kv_metrics_logging.py", ) +indexer_patch = load_module("indexer_barrier_patch", HERE / "patch_indexer_barrier.py") + + +def test_indexer_transform_rejects_unknown_source_without_writing(tmp_path): + target = tmp_path / "fused_indexer.py" + target.write_bytes(b"unknown source\n") + with pytest.raises(RuntimeError, match="unsupported B12X"): + indexer_patch.apply_patch(target) + assert target.read_bytes() == b"unknown source\n" + + +def test_indexer_transform_applies_both_changes_and_is_idempotent(tmp_path, monkeypatch): + before = (indexer_patch._BEFORE + indexer_patch._OLD_CACHE).encode() + after = (indexer_patch._AFTER + indexer_patch._NEW_CACHE).encode() + monkeypatch.setattr(indexer_patch, "BEFORE_SHA256", hashlib.sha256(before).hexdigest()) + monkeypatch.setattr(indexer_patch, "AFTER_SHA256", hashlib.sha256(after).hexdigest()) + target = tmp_path / "fused_indexer.py" + target.write_bytes(before) + indexer_patch.apply_patch(target) + assert target.read_bytes() == after + indexer_patch.apply_patch(target) + assert target.read_bytes() == after + + +def test_indexer_transform_rejects_wrong_postimage_without_writing(tmp_path, monkeypatch): + before = (indexer_patch._BEFORE + indexer_patch._OLD_CACHE).encode() + monkeypatch.setattr(indexer_patch, "BEFORE_SHA256", hashlib.sha256(before).hexdigest()) + target = tmp_path / "fused_indexer.py" + target.write_bytes(before) + with pytest.raises(RuntimeError, match="GPU-tested source"): + indexer_patch.apply_patch(target) + assert target.read_bytes() == before def test_metrics_patch_uses_connector_owned_compact_lines(tmp_path: Path) -> None: diff --git a/runtime/glm53-flash-jj-r8-gb10/test_launcher_contract.py b/runtime/glm53-flash-jj-r8-gb10/test_launcher_contract.py index e77290a0..333e5798 100644 --- a/runtime/glm53-flash-jj-r8-gb10/test_launcher_contract.py +++ b/runtime/glm53-flash-jj-r8-gb10/test_launcher_contract.py @@ -163,6 +163,8 @@ def test_launcher_resolves_dcp_profiles_and_prompt_token_details( assert result.returncode == 0, result.stderr arguments = capture.read_text(encoding="utf-8").splitlines() assert "test-image:r8" in arguments + assert "SPARKRING_LIVENESS_OUTPUT_SECONDS=300" in arguments + assert "B12X_FUSED_INDEXER=1" in arguments dcp_index = arguments.index("--decode-context-parallel-size") assert arguments[dcp_index + 1] == str(dcp) interleave_index = arguments.index("--cp-kv-cache-interleave-size") @@ -225,6 +227,17 @@ def test_launcher_resolves_dcp_profiles_and_prompt_token_details( ) assert unchanged.returncode == 0, unchanged.stderr assert capture.read_text(encoding="utf-8").splitlines() == arguments + config.write_text(original_config + "\nB12X_FUSED_INDEXER=0\n", + encoding="utf-8", newline="\n") + indexer_override = subprocess.run( + ["bash", _bash_path(LAUNCHER), "0", _bash_path(config)], + cwd=ROOT, text=True, capture_output=True, check=False, + ) + assert indexer_override.returncode == 0, indexer_override.stderr + expected_indexer_override = arguments.copy() + indexer_env = expected_indexer_override.index("B12X_FUSED_INDEXER=1") + expected_indexer_override[indexer_env] = "B12X_FUSED_INDEXER=0" + assert capture.read_text(encoding="utf-8").splitlines() == expected_indexer_override config = tmp_path / "dcp1-vllm-prefix-only.env" config.write_text( diff --git a/runtime/glm53-flash-jj-r8-gb10/test_scheduler_liveness.py b/runtime/glm53-flash-jj-r8-gb10/test_scheduler_liveness.py index 8a83f54a..63dda740 100644 --- a/runtime/glm53-flash-jj-r8-gb10/test_scheduler_liveness.py +++ b/runtime/glm53-flash-jj-r8-gb10/test_scheduler_liveness.py @@ -4,6 +4,8 @@ import json from pathlib import Path +import pytest + HERE = Path(__file__).resolve().parent @@ -143,3 +145,102 @@ def test_initial_unavailable_snapshot_is_strict_json() -> None: assert snapshot["healthy"] is False assert snapshot["sample_age_seconds"] is None json.dumps(snapshot, allow_nan=False) + + +def _output_metrics(*, running=3, iterations=10): + return _metrics(running=running, waiting=0, kv=0.2) + ( + f'\nvllm:iteration_tokens_total_count{{engine="0"}} {iterations}' + ) + + +def test_fresh_scrapes_do_not_hide_running_output_stall() -> None: + module = _load_module() + now = [100.0] + monitor = module.SchedulerLiveness( + blocked_timeout_seconds=60, idle_kv_warn_seconds=330, + stale_sample_seconds=15, clock=lambda: now[0], + ) + monitor.observe(_output_metrics()) + for elapsed in range(10, 301, 10): + now[0] = 100.0 + elapsed + monitor.observe(_output_metrics()) + assert monitor.http_status() == 503 + assert monitor.snapshot()["reason"] == "engine_output_stall" + assert monitor.snapshot()["output_stalled_seconds"] == 300 + + +def test_output_progress_and_idle_restart_the_stall_window() -> None: + module = _load_module() + now = [0.0] + monitor = module.SchedulerLiveness( + blocked_timeout_seconds=60, idle_kv_warn_seconds=330, + stale_sample_seconds=15, clock=lambda: now[0], + ) + for timestamp, running, iterations in ( + (0, 3, 10), (290, 3, 11), (580, 3, 12), + (870, 0, 12), (2000, 3, 12), (2290, 3, 12), + (2300, 3, 1), (2590, 3, 1), + ): + now[0] = timestamp + monitor.observe(_output_metrics(running=running, iterations=iterations)) + assert monitor.http_status() == 200 + + +def test_running_stall_timeout_is_independent_of_capacity_timeout() -> None: + module = _load_module() + now = [0.0] + monitor = module.SchedulerLiveness( + blocked_timeout_seconds=60, output_timeout_seconds=900, + idle_kv_warn_seconds=330, stale_sample_seconds=15, + clock=lambda: now[0], + ) + monitor.observe(_output_metrics()) + now[0] = 899 + monitor.observe(_output_metrics()) + assert monitor.http_status() == 200 + now[0] = 900 + monitor.observe(_output_metrics()) + assert monitor.http_status() == 503 + assert "sparkring:engine_output_stalled_seconds 900" in monitor.prometheus() + + +def test_missing_progress_metric_does_not_refresh_sample() -> None: + module = _load_module() + now = [0.0] + monitor = module.SchedulerLiveness( + blocked_timeout_seconds=60, idle_kv_warn_seconds=330, + stale_sample_seconds=15, clock=lambda: now[0], + ) + monitor.observe(_output_metrics()) + now[0] = 16 + with pytest.raises(ValueError, match="iteration_tokens_total_count") as error: + monitor.observe(_metrics(running=3, waiting=0, kv=0.2)) + monitor.observe_error(error.value) + assert monitor.snapshot()["reason"] == "metrics_unavailable" + assert monitor.http_status() == 503 + + +@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan")]) +def test_invalid_output_timeout_is_rejected(timeout) -> None: + with pytest.raises(ValueError, match="output timeout"): + _load_module().SchedulerLiveness( + blocked_timeout_seconds=60, output_timeout_seconds=timeout, + idle_kv_warn_seconds=330, stale_sample_seconds=15, + ) + + +def test_progress_recovers_unhealthy_monitor() -> None: + module = _load_module() + now = [0.0] + monitor = module.SchedulerLiveness( + blocked_timeout_seconds=60, idle_kv_warn_seconds=330, + stale_sample_seconds=15, clock=lambda: now[0], + ) + monitor.observe(_output_metrics()) + now[0] = 300 + monitor.observe(_output_metrics()) + assert monitor.http_status() == 503 + now[0] += 10 + monitor.observe(_output_metrics(iterations=11)) + assert monitor.http_status() == 200 + assert monitor.snapshot()["output_stalled_seconds"] == 0 diff --git a/runtime/glm53-flash-jj-r8-gb10/test_serve_with_warmup.py b/runtime/glm53-flash-jj-r8-gb10/test_serve_with_warmup.py index 1f92677b..fd4cae4b 100644 --- a/runtime/glm53-flash-jj-r8-gb10/test_serve_with_warmup.py +++ b/runtime/glm53-flash-jj-r8-gb10/test_serve_with_warmup.py @@ -134,6 +134,7 @@ def test_rank_zero_starts_scheduler_liveness_service(monkeypatch) -> None: monkeypatch.setenv("SPARKRING_LIVENESS_ENABLED", "1") monkeypatch.setenv("SPARKRING_LIVENESS_PORT", "9016") monkeypatch.setenv("SPARKRING_LIVENESS_BLOCKED_SECONDS", "75") + monkeypatch.setenv("SPARKRING_LIVENESS_OUTPUT_SECONDS", "900") monkeypatch.setenv("SPARKRING_IDLE_KV_WARN_SECONDS", "360") monkeypatch.setenv("SPARKRING_LIVENESS_STALE_SECONDS", "20") monkeypatch.setenv("SPARKRING_LIVENESS_SAMPLE_SECONDS", "3") @@ -150,6 +151,7 @@ def test_rank_zero_starts_scheduler_liveness_service(monkeypatch) -> None: "metrics_url": "http://127.0.0.1:8015/metrics", "port": 9016, "blocked_timeout_seconds": 75.0, + "output_timeout_seconds": 900.0, "idle_kv_warn_seconds": 360.0, "stale_sample_seconds": 20.0, "sample_interval_seconds": 3.0, From 5415f5058c1616f0173dc251be9cc36f9514f07c Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:27:13 -0500 Subject: [PATCH 10/16] Exercise sampling and reasoning before GLM readiness --- runtime/glm53-flash-jj-r8-gb10/README.md | 7 +++ .../serve_with_warmup.py | 45 ++++++++++++++++++ .../test_serve_with_warmup.py | 47 ++++++++++++++++++- 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/runtime/glm53-flash-jj-r8-gb10/README.md b/runtime/glm53-flash-jj-r8-gb10/README.md index 8076bcaf..3661fb9e 100644 --- a/runtime/glm53-flash-jj-r8-gb10/README.md +++ b/runtime/glm53-flash-jj-r8-gb10/README.md @@ -288,6 +288,13 @@ arena wait that would justify more unified-memory pressure. The image entrypoint runs `warmup_dflash.py` before Docker reports rank 0 as healthy. +The readiness wrapper `serve_with_warmup.py` includes a final temperature-one request with thinking +enabled, in addition to the configured shape batches. Failure of that request +prevents warmup completion. This sampling coverage is implemented with CPU +request-contract tests; kernel coverage requires a rebuilt image and GPU +validation. It does not establish coverage of mixed long/short prefill batches +or all recurrent KDA specializations. The published image receipt does not +qualify the additional request. The default environment template warms every concurrency from C1 through C16 and prompt spans covering the DFlash Triton `BLOCK_SIZE` specializations through 256. DFlash depth seven verifies eight target rows per active request, diff --git a/runtime/glm53-flash-jj-r8-gb10/serve_with_warmup.py b/runtime/glm53-flash-jj-r8-gb10/serve_with_warmup.py index 3ec746c4..2ee8b0ed 100644 --- a/runtime/glm53-flash-jj-r8-gb10/serve_with_warmup.py +++ b/runtime/glm53-flash-jj-r8-gb10/serve_with_warmup.py @@ -8,6 +8,8 @@ import signal import subprocess import sys +import time +import urllib.request from pathlib import Path import warmup_dflash @@ -17,6 +19,45 @@ READY_PATH = Path("/tmp/sparkring-engine-ready") +def warmup_sampling( + endpoint: str, + model: str, + max_tokens: int, + timeout_seconds: float, + credential: str | None, +) -> dict[str, object]: + """Exercise stochastic sampling and reasoning before declaring readiness.""" + body = { + "model": model, + "messages": [{ + "role": "user", + "content": f"Sampling warmup {time.monotonic_ns()}. Reply briefly.", + }], + "temperature": 1.0, + "max_tokens": max_tokens, + "chat_template_kwargs": {"enable_thinking": True}, + } + headers = {"Content-Type": "application/json"} + if credential: + headers["Authorization"] = f"Bearer {credential}" + request = urllib.request.Request( + endpoint.rstrip("/") + "/v1/chat/completions", + data=json.dumps(body).encode(), + headers=headers, + ) + started = time.monotonic() + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: + result = json.load(response) + choices = result.get("choices") + if not isinstance(choices, list) or not choices: + raise RuntimeError("Sampling warmup response has no completion") + return { + "temperature": 1.0, + "enable_thinking": True, + "elapsed_seconds": round(time.monotonic() - started, 3), + } + + def _positive_csv(value: str, name: str) -> tuple[int, ...]: try: result = tuple(int(item) for item in value.split(",")) @@ -56,6 +97,10 @@ def complete_readiness( shape_words, credential, ) + sampling = warmup_sampling( + endpoint, model, max_tokens, timeout_seconds, credential + ) + print(json.dumps({"sampling_warmup": sampling}, separators=(",", ":"))) print(json.dumps({"dflash_warmup": result}, separators=(",", ":"))) ready_path.touch() diff --git a/runtime/glm53-flash-jj-r8-gb10/test_serve_with_warmup.py b/runtime/glm53-flash-jj-r8-gb10/test_serve_with_warmup.py index fd4cae4b..406f4a6e 100644 --- a/runtime/glm53-flash-jj-r8-gb10/test_serve_with_warmup.py +++ b/runtime/glm53-flash-jj-r8-gb10/test_serve_with_warmup.py @@ -1,6 +1,8 @@ from __future__ import annotations import importlib.util +import io +import json import sys from pathlib import Path @@ -48,6 +50,10 @@ def run(*_args): return ({"concurrency": 2},) monkeypatch.setattr(warmup, "run_warmup", run) + monkeypatch.setattr( + wrapper, "warmup_sampling", + lambda *_args: events.append("sampling") or {}, + ) wrapper.complete_readiness( rank=0, @@ -62,10 +68,49 @@ def run(*_args): ready_path=ready, ) - assert events == ["api", "warmup"] + assert events == ["api", "warmup", "sampling"] assert ready.is_file() +def test_sampling_request_uses_auth_temperature_and_reasoning(monkeypatch): + wrapper, _ = _load_module(monkeypatch) + observed = [] + + def urlopen(request, timeout): + observed.append(request) + assert timeout == 10 + return io.BytesIO(b'{"choices":[{"message":{"reasoning":"ok"}}]}') + + monkeypatch.setattr(wrapper.urllib.request, "urlopen", urlopen) + result = wrapper.warmup_sampling("http://localhost/", "model", 16, 10, "secret") + body = json.loads(observed[0].data) + assert observed[0].full_url == "http://localhost/v1/chat/completions" + assert observed[0].get_header("Authorization") == "Bearer secret" + assert body["temperature"] == 1.0 + assert body["chat_template_kwargs"] == {"enable_thinking": True} + assert body["max_tokens"] == 16 + assert result["enable_thinking"] is True + + +def test_sampling_failure_withholds_readiness_marker(tmp_path, monkeypatch): + import pytest + + wrapper, warmup = _load_module(monkeypatch) + ready = tmp_path / "ready" + ready.touch() + monkeypatch.setattr(warmup, "wait_for_api", lambda *_args: None) + monkeypatch.setattr(warmup, "run_warmup", lambda *_args: ()) + monkeypatch.setattr(wrapper.urllib.request, "urlopen", + lambda *_args, **_kwargs: io.BytesIO(b'{"choices":[]}')) + with pytest.raises(RuntimeError, match="Sampling warmup response has no completion"): + wrapper.complete_readiness( + rank=0, endpoint="http://localhost", model="model", + warmup_enabled=True, concurrencies=(1,), shape_words=(8,), + max_tokens=16, timeout_seconds=10, credential=None, ready_path=ready, + ) + assert not ready.exists() + + def test_headless_rank_does_not_call_http_warmup(tmp_path: Path, monkeypatch) -> None: wrapper, warmup = _load_module(monkeypatch) ready = tmp_path / "ready" From ef3c381bd41eef07abcb5daebfc9c81d5928ff88 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:35:51 -0500 Subject: [PATCH 11/16] Package a checked child image for indexer and readiness fixes --- .../glm53-flash-jj-r8-gb10/hotfix/Dockerfile | 14 +++ .../hotfix/install_hotfix.py | 96 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 runtime/glm53-flash-jj-r8-gb10/hotfix/Dockerfile create mode 100644 runtime/glm53-flash-jj-r8-gb10/hotfix/install_hotfix.py diff --git a/runtime/glm53-flash-jj-r8-gb10/hotfix/Dockerfile b/runtime/glm53-flash-jj-r8-gb10/hotfix/Dockerfile new file mode 100644 index 00000000..f8ce416d --- /dev/null +++ b/runtime/glm53-flash-jj-r8-gb10/hotfix/Dockerfile @@ -0,0 +1,14 @@ +ARG PARENT_IMAGE=ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:0d4029b3b7023cf32c37ac20279469c9a2ee16a057f25aae3bcfee9ee5fb660f +FROM ${PARENT_IMAGE} +ARG SOURCE_RECEIPT_SHA256 +ARG SPARKRING_REVISION +COPY patch_indexer_barrier.py install_hotfix.py serve_with_warmup.py scheduler_liveness.py /opt/sparkring/issue224/ +RUN python3 /opt/sparkring/issue224/install_hotfix.py --expected-source-receipt ${SOURCE_RECEIPT_SHA256} +LABEL org.opencontainers.image.revision="${SPARKRING_REVISION}" \ + org.opencontainers.image.base.name="ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:0d4029b3b7023cf32c37ac20279469c9a2ee16a057f25aae3bcfee9ee5fb660f" \ + org.sparkring.parent.image="sha256:5e32aaa1bbe3559e81db7706ed4286248f18d27cfdb186f6b851bf786eb43075" \ + org.sparkring.source-receipt-sha256="${SOURCE_RECEIPT_SHA256}" \ + org.sparkring.runtime.status="implemented-indexer-gpu-tested-no-model-soak" \ + org.sparkring.indexer-barrier.sha256="49f6fd916fd1ccf94311ee99427551edbd0dc3a5de23aeeb426418370f76f66d" \ + org.sparkring.indexer-barrier.issue="https://github.com/FujitsuPolycom/sparkring/issues/224" \ + org.sparkring.sampling-warmup="temperature1-thinking-enabled-before-readiness" diff --git a/runtime/glm53-flash-jj-r8-gb10/hotfix/install_hotfix.py b/runtime/glm53-flash-jj-r8-gb10/hotfix/install_hotfix.py new file mode 100644 index 00000000..b381354f --- /dev/null +++ b/runtime/glm53-flash-jj-r8-gb10/hotfix/install_hotfix.py @@ -0,0 +1,96 @@ +"""Install the checked indexer fix and preserve parent/source receipt provenance.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import shutil + +from patch_indexer_barrier import apply_patch, BEFORE_SHA256, AFTER_SHA256 + + +BASE = Path('/opt/sparkring/overlays/jj-r8-sparkcache-arm64') +RECEIPTS = Path('/opt/sparkring/receipts/jj-r8-sparkcache-arm64') +SITE = Path('/usr/local/lib/python3.12/dist-packages') +RELATIVE = 'b12x/attention/dsa_indexer/fused_indexer.py' +VERIFY = '/opt/sparkring/bin/verify-jj-r8-sparkcache-image.py' + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + '\n', encoding='utf-8') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--expected-source-receipt') + parser.add_argument('--output', type=Path) + args = parser.parse_args() + subprocess.run([sys.executable, VERIFY, '--inside-image'], check=True) + original_receipt = json.loads((RECEIPTS / 'source-receipt.json').read_text()) + wrapper_changes = {} + for name, destination in ( + ('serve_with_warmup.py', '/opt/sparkring/bin/serve-with-warmup.py'), + ('scheduler_liveness.py', '/opt/sparkring/bin/scheduler_liveness.py'), + ): + target = Path(destination) + before = digest(target) + if before != original_receipt['inputs'][name]: + raise RuntimeError(f'parent wrapper differs from its receipt: {name}') + shutil.copyfile(Path(__file__).with_name(name), target) + target.chmod(0o755) + wrapper_changes[name] = {'before_sha256': before, 'after_sha256': digest(target)} + for root in (SITE, BASE / 'sources'): + apply_patch(root / RELATIVE) + for cached in (SITE / RELATIVE).parent.glob('__pycache__/fused_indexer.*.pyc'): + cached.unlink() + source_before = digest(RECEIPTS / 'source-receipt.json') + transform = digest(Path(__file__).with_name('patch_indexer_barrier.py')) + for root in (RECEIPTS, BASE / 'receipts'): + manifest_path = root / 'b12x-source-manifest.json' + manifest = json.loads(manifest_path.read_text()) + if manifest['files'][RELATIVE] != BEFORE_SHA256: + raise RuntimeError('parent source manifest differs from the tested preimage') + manifest['files'][RELATIVE] = AFTER_SHA256 + write_json(manifest_path, manifest) + receipt_path = root / 'source-receipt.json' + original = receipt_path.read_bytes() + receipt_path.with_name('source-receipt.parent.json').write_bytes(original) + receipt = json.loads(original) + receipt['parent_source_receipt_sha256'] = source_before + receipt['inputs']['bundle/receipts/b12x-source-manifest.json'] = digest(manifest_path) + receipt['inputs']['source_transform/indexer_barrier'] = transform + for name, change in wrapper_changes.items(): + receipt['inputs'][name] = change['after_sha256'] + receipt['source_transforms'] = {'indexer_barrier': { + 'path': RELATIVE, 'before_sha256': BEFORE_SHA256, + 'after_sha256': AFTER_SHA256, 'transform_sha256': transform, + }} + write_json(receipt_path, receipt) + source_after = digest(RECEIPTS / 'source-receipt.json') + if args.expected_source_receipt and source_after != args.expected_source_receipt: + raise RuntimeError('source receipt differs from the prepared build input') + result = { + 'schema': 'sparkring-indexer-barrier-hotfix/v1', + 'parent_image_id': 'sha256:5e32aaa1bbe3559e81db7706ed4286248f18d27cfdb186f6b851bf786eb43075', + 'parent_registry_digest': 'ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:0d4029b3b7023cf32c37ac20279469c9a2ee16a057f25aae3bcfee9ee5fb660f', + 'source_before_sha256': BEFORE_SHA256, 'source_after_sha256': AFTER_SHA256, + 'parent_source_receipt_sha256': source_before, + 'source_receipt_sha256': source_after, 'transform_sha256': transform, + 'installer_sha256': digest(Path(__file__)), + 'wrapper_changes': wrapper_changes, + 'scope': 'indexer barrier/compile revision, sampling readiness warmup, output-stall liveness; native libraries unchanged', + } + write_json(RECEIPTS / 'issue224-hotfix.json', result) + subprocess.run([sys.executable, VERIFY, '--inside-image'], check=True) + if args.output: + write_json(args.output, result) + print(json.dumps(result), flush=True) + + +if __name__ == '__main__': + main() From d18be4ca074d1eb41299d1b39846f84f8046a104 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:40:23 -0500 Subject: [PATCH 12/16] Record published indexer and readiness hotfix image --- docs/ISSUE224_INDEXER_BARRIER.md | 5 +- runtime/glm53-flash-jj-r8-gb10/README.md | 5 +- .../glm53-flash-jj-r8-gb10/hotfix/README.md | 61 +++++++++++++++++++ .../hotfix/public-image.json | 48 +++++++++++++++ 4 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 runtime/glm53-flash-jj-r8-gb10/hotfix/README.md create mode 100644 runtime/glm53-flash-jj-r8-gb10/hotfix/public-image.json diff --git a/docs/ISSUE224_INDEXER_BARRIER.md b/docs/ISSUE224_INDEXER_BARRIER.md index c19ef1db..c79a7c51 100644 --- a/docs/ISSUE224_INDEXER_BARRIER.md +++ b/docs/ISSUE224_INDEXER_BARRIER.md @@ -164,8 +164,9 @@ unexpected source rather than applying a speculative replacement. The output uses LF line endings; the original GPU test file used mixed line endings. Both have identical Python source after newline normalization. -Existing published image digests are unchanged. Rebuild with this change to -include the fix; restarting an existing image alone does not install it. +The [published child image](../runtime/glm53-flash-jj-r8-gb10/hotfix/README.md) +includes the fix; users do not need to rebuild it. Existing image digests are +unchanged, so restarting an old image alone does not install the update. For a targeted A/B run, pass `B12X_FUSED_INDEXER=0` inside every worker container before startup. The pinned `dsa_indexer/scratch.py` recognizes this switch and diff --git a/runtime/glm53-flash-jj-r8-gb10/README.md b/runtime/glm53-flash-jj-r8-gb10/README.md index 3661fb9e..d71b35f6 100644 --- a/runtime/glm53-flash-jj-r8-gb10/README.md +++ b/runtime/glm53-flash-jj-r8-gb10/README.md @@ -396,8 +396,9 @@ separate JIT cache namespace and coordinated restart on all ranks. This bypass can change throughput and has not been qualified on the affected cluster. The image builder applies the GPU-tested publication barrier and compile-cache revision through `patch_indexer_barrier.py` before generating the B12X source -manifest. This requires rebuilding the image; published image pins remain -unchanged. See [the source trace and fix](../../docs/ISSUE224_INDEXER_BARRIER.md). +manifest. A [published child image](hotfix/README.md) includes this correction, +sampling/reasoning readiness warmup, and output-stall detection. The default +image pins remain unchanged. See [the source trace and fix](../../docs/ISSUE224_INDEXER_BARRIER.md). Idle KV retention is warning-only. The default 330-second warning interval is longer than the GLM profile's 300-second shared-prefix lease, so an intentional diff --git a/runtime/glm53-flash-jj-r8-gb10/hotfix/README.md b/runtime/glm53-flash-jj-r8-gb10/hotfix/README.md new file mode 100644 index 00000000..595fb7b1 --- /dev/null +++ b/runtime/glm53-flash-jj-r8-gb10/hotfix/README.md @@ -0,0 +1,61 @@ +# Indexer and sampling-readiness image update + +Status: implemented and GPU-tested. No full-model soak or serving replacement +is claimed. This is a small child of the published SIRCL operator image, with +the tested B12X publication barrier, its new compile revision, output-stall +liveness detection, and temperature-one/thinking-enabled readiness warmup. + +Published tag: +`ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache:20260906-indexer-barrier-warmup` + +Immutable reference: +`ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:28e6a9c0dba07cec4852bf21352e5e2f6fc7bd07592a0edc3916d13f849cdcfe` + +Local/config image ID: +`sha256:d2ba5894bd5499883acbb89ecb496965c44f78aa3a4b4be83c4698b2010ec787` + +For the operator launcher, set `IMAGE_REF` to the immutable reference and +`IMAGE_ID` to the config image ID on every rank. A distinct +`JIT_CACHE_NAMESPACE`, such as `glm53-20260906-indexer-barrier-warmup`, keeps +the update's warmup evidence separate. The sampling request runs when +`DFLASH_WARMUP=1`; disabled warmup and headless-rank behavior are unchanged. +Use the deployment's coordinated stop/start procedure to install the update. + +This image replaces the SIRCL operator parent whose digest starts `0d4029b3`. +It does not contain the additional managed-MTP3 mesh bundle from the separate +image whose digest starts `23f00af8`. Preserve that bundle when rebuilding a +managed mesh child; do not substitute this base tag for a managed mesh image. +The DeepSeek-specific #217 serializer patch is not part of this GLM runtime. + +## Checks on the built image + +- Source manifests and all retained native libraries verified before and after + the patch. The updated source receipt records the transform and wrapper + hashes; the original receipt is retained as parent provenance. +- 22 tests exercised the installed readiness and liveness modules. +- 15 full-indexer GPU correctness tests passed on GB10. +- 2,000 GLM-shaped graph replays passed with exact selected-index parity, + contexts up to 200K tokens, and concurrent 64 MiB GPU copies. +- Anonymous registry pull succeeded on another DGX4 node. +- The image adds two layers and 428,386 uncompressed bytes to its parent. + +The warmup tests validate the installed request/marker contract, not cold-cache +full-model specialization coverage. The long-running TP4/SparkCache workload +was not repeated. Report recurrence with the image digest, runtime settings, +EngineCore stack, and all worker-thread stacks. + +## Rebuild + +Assemble an empty context with this `Dockerfile` and `install_hotfix.py`, plus +`patch_indexer_barrier.py`, `serve_with_warmup.py`, and `scheduler_liveness.py` +from the parent runtime directory. Use LF line endings. Run the installer in +an isolated container of the pinned parent with that context mounted at +`/hotfix`, passing `--output /hotfix/build-input.json`. + +Build with the resulting `source_receipt_sha256` as the +`SOURCE_RECEIPT_SHA256` build argument and the source checkout commit as +`SPARKRING_REVISION`. The installer rejects unexpected parent source or +wrapper hashes and verifies the final source receipt against the build argument. +The published build uses source commit +`ef3c381bd41eef07abcb5daebfc9c81d5928ff88` and source receipt +`b27290a28e6d322d37b9cea11b01cd76d18a83f2dd58e6f4e1094cdc83c9b4bc`. diff --git a/runtime/glm53-flash-jj-r8-gb10/hotfix/public-image.json b/runtime/glm53-flash-jj-r8-gb10/hotfix/public-image.json new file mode 100644 index 00000000..9d54d648 --- /dev/null +++ b/runtime/glm53-flash-jj-r8-gb10/hotfix/public-image.json @@ -0,0 +1,48 @@ +{ + "artifact": { + "added_layers": 2, + "added_uncompressed_bytes": 428386, + "image_id": "sha256:d2ba5894bd5499883acbb89ecb496965c44f78aa3a4b4be83c4698b2010ec787", + "platform": "linux/arm64", + "reference": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:28e6a9c0dba07cec4852bf21352e5e2f6fc7bd07592a0edc3916d13f849cdcfe", + "size_bytes": 21075186765, + "source_revision": "ef3c381bd41eef07abcb5daebfc9c81d5928ff88", + "tag": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache:20260906-indexer-barrier-warmup" + }, + "installer_sha256": "5597b51f70fcc16c3c1097370fbcf1659aadfd7c5a01f0d4494e814ad48cc368", + "limitations": [ + "Original SIRCL operator parent only; managed-MTP mesh bundle not included.", + "DeepSeek issue 217 is a separate runtime and is not included.", + "Sampling warmup request contract tested; no full-model cold-cache specialization qualification." + ], + "parent_image_id": "sha256:5e32aaa1bbe3559e81db7706ed4286248f18d27cfdb186f6b851bf786eb43075", + "parent_registry_digest": "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:0d4029b3b7023cf32c37ac20279469c9a2ee16a057f25aae3bcfee9ee5fb660f", + "parent_source_receipt_sha256": "6ef1ddd37fda9b797a8c71473db284c7d79f206207e43b38adc1ba5a1faa4359", + "schema": "sparkring-indexer-barrier-hotfix/v1", + "scope": "indexer barrier/compile revision, sampling readiness warmup, output-stall liveness; native libraries unchanged", + "source_after_sha256": "49f6fd916fd1ccf94311ee99427551edbd0dc3a5de23aeeb426418370f76f66d", + "source_before_sha256": "d3ec6274e142a4e7d1062ea6d2d99b97db0a02e92bb976c6570ae990b836b18d", + "source_receipt_sha256": "b27290a28e6d322d37b9cea11b01cd76d18a83f2dd58e6f4e1094cdc83c9b4bc", + "transform_sha256": "d4721099070b23ecc80fbaa5de2a378578b039ca5b78d3d83446e295ad3260aa", + "verification": { + "anonymous_pull": true, + "full_model_soak": false, + "gpu_correctness_tests_passed": 15, + "graph_replays_passed": 2000, + "packaged_wrapper_tests_passed": 22, + "pulled_image_id_matched": true, + "pulled_source_and_native_verification": true, + "serving_containers_replaced": false, + "source_and_native_manifests": true + }, + "wrapper_changes": { + "scheduler_liveness.py": { + "after_sha256": "6fe1f8e052577c7a71708236bcd452ca3377f99de776e7edef8881573dac5670", + "before_sha256": "507e103f69ea9add095f9c8c0e45baf736ba1a25bf175bf7900eaee3f7f06b2a" + }, + "serve_with_warmup.py": { + "after_sha256": "7579c43090dece26792bca6494f48afc9ce6044051426e38e14882d36805d6b2", + "before_sha256": "3593e2cf637393e6905a0e37e8bf5a58a718fee01d89892d040d165ffc4c2509" + } + } +} From d1ec3ec63607bb6211ee2e9d21381f54f496b05d Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 6 Sep 2026 01:04:00 -0500 Subject: [PATCH 13/16] Specify operator image verification and reproducible barrier checks --- docs/ISSUE224_ENGINE_STALL.md | 13 ++++++------ docs/ISSUE224_INDEXER_BARRIER.md | 20 +++++++++++-------- runtime/glm53-flash-jj-r8-gb10/README.md | 19 +++++++++--------- .../glm53-flash-jj-r8-gb10/hotfix/README.md | 16 +++++++++++++++ 4 files changed, 45 insertions(+), 23 deletions(-) diff --git a/docs/ISSUE224_ENGINE_STALL.md b/docs/ISSUE224_ENGINE_STALL.md index c87565bd..6e92e6e2 100644 --- a/docs/ISSUE224_ENGINE_STALL.md +++ b/docs/ISSUE224_ENGINE_STALL.md @@ -74,12 +74,13 @@ progress, idle periods, counter reset, recovery, missing metrics, invalid timeouts, and independent prefill grace. No GPU race is reproduced by these tests. -Rebuild the operator wrapper to deploy the monitor; editing a runtime setting -alone cannot update an existing image. This patch does not change published -image pins or qualification receipts. Validate the rebuilt image with both -long healthy prefills/restores and injected output stalls before using its -signal for unattended recovery. Follow the managed deployment's coordinated -stop/recovery procedure; this monitor only reports health. +The [published operator image](../runtime/glm53-flash-jj-r8-gb10/hotfix/README.md) +contains the monitor and its source receipt. Install that immutable image with +the deployment's coordinated stop/start procedure; users do not need to rebuild +it. Editing a setting alone cannot update an image that lacks the monitor. +Unattended recovery remains unqualified: the deployment must first establish +its longest healthy prefill/restore gap and verify its response to an injected +output stall. The monitor reports health and does not restart the cluster. Do not automatically replay a timed-out model step. The executor's responses are ordered without per-call IDs, and a partially completed step can mutate KV diff --git a/docs/ISSUE224_INDEXER_BARRIER.md b/docs/ISSUE224_INDEXER_BARRIER.md index c79a7c51..a6794101 100644 --- a/docs/ISSUE224_INDEXER_BARRIER.md +++ b/docs/ISSUE224_INDEXER_BARRIER.md @@ -62,18 +62,22 @@ AST-lowered copy of the actual barrier helper; CUDA scheduling and code generati are not executed. This is evidence of a reachable protocol failure, not a hardware reproduction or a measurement of its production frequency. -The accompanying standalone model uses only Python's standard library. Run it -against a clean checkout of the affected B12X pin, then apply the candidate patch +The CPU interleaving harness, +[`repro_indexer_barrier.py`](../performance/harnesses/indexer_barrier/repro_indexer_barrier.py), +uses only Python's standard library. From the SparkRing repository root, run it +against an LF checkout of B12X commit +`9ae41c5cb9935d740456479954b0089f80bd2ef2`, apply the checked source transform, and repeat: ```bash -python repro_indexer_barrier.py /path/to/b12x/b12x/attention/dsa_indexer/fused_indexer.py --expect deadlock -git -C /path/to/b12x apply /path/to/issue224-b12x-barrier.patch -python repro_indexer_barrier.py /path/to/b12x/b12x/attention/dsa_indexer/fused_indexer.py --expect complete +python performance/harnesses/indexer_barrier/repro_indexer_barrier.py /path/to/b12x/b12x/attention/dsa_indexer/fused_indexer.py --expect deadlock +python runtime/glm53-flash-jj-r8-gb10/patch_indexer_barrier.py /path/to/b12x/b12x/attention/dsa_indexer/fused_indexer.py +python performance/harnesses/indexer_barrier/repro_indexer_barrier.py /path/to/b12x/b12x/attention/dsa_indexer/fused_indexer.py --expect complete ``` -The evidence JSON records both traces: before the change, arrival is 4 with a -block waiting for 6; after the change, arrival reaches 6 with no pending actors. +The harness prints a JSON trace. Without entry synchronization, the modeled +arrival count is 4 while a block waits for 6. With entry synchronization, the +count reaches 6 and no actors remain pending. ## Full response path @@ -132,7 +136,7 @@ and query-row count are not interchangeable, especially with speculation. This makes the failure path reachable under the recorded default composition. The exact live plan and group shape at each reported hang are not available. -## Corrections to the original diagnosis +## Executor response deadlines The exact vLLM comparison from `22ffe140` to `e02b1746` leaves the executor, shared-memory queue, and EngineCore files unchanged. Their source already has: diff --git a/runtime/glm53-flash-jj-r8-gb10/README.md b/runtime/glm53-flash-jj-r8-gb10/README.md index d71b35f6..f4f7e247 100644 --- a/runtime/glm53-flash-jj-r8-gb10/README.md +++ b/runtime/glm53-flash-jj-r8-gb10/README.md @@ -286,15 +286,16 @@ NVMe reads and CUDA placement through two 256 MiB mapped arenas. A third arena is not part of the profile because the two-stage pipeline has no measured arena wait that would justify more unified-memory pressure. -The image entrypoint runs `warmup_dflash.py` before Docker reports rank 0 as -healthy. -The readiness wrapper `serve_with_warmup.py` includes a final temperature-one request with thinking -enabled, in addition to the configured shape batches. Failure of that request -prevents warmup completion. This sampling coverage is implemented with CPU -request-contract tests; kernel coverage requires a rebuilt image and GPU -validation. It does not establish coverage of mixed long/short prefill batches -or all recurrent KDA specializations. The published image receipt does not -qualify the additional request. +When `DFLASH_WARMUP=1`, the readiness entrypoint runs `warmup_dflash.py` before +Docker reports rank 0 as healthy. The readiness wrapper, +`serve_with_warmup.py`, also sends a temperature-one request with thinking +enabled after the configured shape batches. Failure of that request prevents +warmup completion. This request contract is implemented and tested in the +[published child image](hotfix/README.md), whose +[receipt](hotfix/public-image.json) records 22 installed readiness/liveness +tests. Cold-cache full-model sampling, mixed long/short prefill coverage, and +all recurrent KDA specializations remain unqualified. The default pinned +operator image predates this additional readiness request. The default environment template warms every concurrency from C1 through C16 and prompt spans covering the DFlash Triton `BLOCK_SIZE` specializations through 256. DFlash depth seven verifies eight target rows per active request, diff --git a/runtime/glm53-flash-jj-r8-gb10/hotfix/README.md b/runtime/glm53-flash-jj-r8-gb10/hotfix/README.md index 595fb7b1..27bf6121 100644 --- a/runtime/glm53-flash-jj-r8-gb10/hotfix/README.md +++ b/runtime/glm53-flash-jj-r8-gb10/hotfix/README.md @@ -44,6 +44,22 @@ full-model specialization coverage. The long-running TP4/SparkCache workload was not repeated. Report recurrence with the image digest, runtime settings, EngineCore stack, and all worker-thread stacks. +## Verify the published child image + +The parent runtime's `verify_image.py --image` command checks the base builder's +labels. The child image has different parent and runtime-status labels, so use +the immutable reference with the installed content verifier: + +```bash +docker run --rm --network none --entrypoint python3 \ + ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:28e6a9c0dba07cec4852bf21352e5e2f6fc7bd07592a0edc3916d13f849cdcfe \ + /opt/sparkring/bin/verify-jj-r8-sparkcache-image.py --inside-image +``` + +This command verifies the image's source manifests and retained native-library +hashes without GPU access or a model process. Its result does not establish +serving qualification. + ## Rebuild Assemble an empty context with this `Dockerfile` and `install_hotfix.py`, plus From cb5471d52bdf0903db2976cbc32767c6d8f9fcb5 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 6 Sep 2026 01:04:02 -0500 Subject: [PATCH 14/16] Describe snapshot blob links without assuming every file is a symlink --- scripts/deepseek_v4_cycle_serve.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/deepseek_v4_cycle_serve.sh b/scripts/deepseek_v4_cycle_serve.sh index f62c3d91..7ab63bf7 100755 --- a/scripts/deepseek_v4_cycle_serve.sh +++ b/scripts/deepseek_v4_cycle_serve.sh @@ -140,12 +140,10 @@ container_name="deepseek-v4-flash-r$NODE_RANK" model_container_path=/models/deepseek-v4-flash-0731 served_model_name=${SERVED_MODEL_NAME:-deepseek-v4-flash-0731} -# HuggingFace hub snapshot support: when MODEL_HOST_PATH points into an HF -# hub cache (`/snapshots//`), every model file inside is a -# symlink whose relative target ../../blobs/ resolves above the mounted -# snapshot tree. Bind the sibling blobs directory read-only so the container -# resolves those targets to the real weight payloads. Plain checkpoint -# directories are unaffected and gain no extra mount. +# Hugging Face snapshot files can link to ../../blobs/, outside the +# mounted snapshot directory. Mount the sibling blobs directory read-only at +# /blobs so those links resolve from /models/deepseek-v4-flash-0731. Plain +# checkpoint directories do not require the additional mount. model_blobs_path= case "/$MODEL_HOST_PATH/" in */snapshots/*/) From c753c0791344bd145a98d978024b62cd2bda99e0 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 6 Sep 2026 01:05:20 -0500 Subject: [PATCH 15/16] Identify the operator image that includes sampling readiness --- runtime/glm53-flash-jj-r8-gb10/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/runtime/glm53-flash-jj-r8-gb10/README.md b/runtime/glm53-flash-jj-r8-gb10/README.md index f4f7e247..95bfd2c4 100644 --- a/runtime/glm53-flash-jj-r8-gb10/README.md +++ b/runtime/glm53-flash-jj-r8-gb10/README.md @@ -294,8 +294,9 @@ warmup completion. This request contract is implemented and tested in the [published child image](hotfix/README.md), whose [receipt](hotfix/public-image.json) records 22 installed readiness/liveness tests. Cold-cache full-model sampling, mixed long/short prefill coverage, and -all recurrent KDA specializations remain unqualified. The default pinned -operator image predates this additional readiness request. +all recurrent KDA specializations remain unqualified. The operator image +referenced by `pins.json` at `operator_image.reference` does not include this +additional readiness request. The default environment template warms every concurrency from C1 through C16 and prompt spans covering the DFlash Triton `BLOCK_SIZE` specializations through 256. DFlash depth seven verifies eight target rows per active request, From 50d46f703c39bdce929a9dccb5cd3cbe534ede99 Mon Sep 17 00:00:00 2001 From: FujitsuPolycom <87842395+FujitsuPolycom@users.noreply.github.com> Date: Sun, 6 Sep 2026 13:41:26 -0500 Subject: [PATCH 16/16] Constrain compute verification paths and encode receipts explicitly Reject absolute, traversing, non-vLLM, and symlinked override paths before reading package bytes. Read JSON as UTF-8 and write receipts as UTF-8 with LF endings. Cache and compute source-lock identities are unchanged; published images are not rebuilt. Validation: five failing path regressions now pass; 16 compute tests and 353 mesh tests pass with one optional native-bundle skip. --- .../compute/apply_compute.py | 9 ++++--- .../compute/prepare_compute_source.py | 4 +-- .../compute/test_compute.py | 25 +++++++++++++++++++ .../compute/verify_compute.py | 16 +++++++++--- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/runtime/glm53-spark-mtp3-mesh/compute/apply_compute.py b/runtime/glm53-spark-mtp3-mesh/compute/apply_compute.py index 95311dad..6dda7627 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/apply_compute.py +++ b/runtime/glm53-spark-mtp3-mesh/compute/apply_compute.py @@ -23,8 +23,8 @@ def _map_sha256(files: dict[str, str]) -> str: def _load(prepared: Path) -> tuple[dict, dict]: lock_path = prepared / "source-lock.json" - lock = json.loads(lock_path.read_text()) - manifest = json.loads((prepared / "prepared-manifest.json").read_text()) + lock = json.loads(lock_path.read_text(encoding="utf-8")) + manifest = json.loads((prepared / "prepared-manifest.json").read_text(encoding="utf-8")) if hashlib.sha256(lock_path.read_bytes()).hexdigest() != manifest[ "source_lock_sha256" ]: @@ -49,7 +49,8 @@ def _install_cuda(prepared: Path, lock: dict, destination: Path) -> None: raise ValueError(f"invalid CUDA archive root: {archive.name}") shutil.copytree(entries[0], destination, dirs_exist_ok=True, symlinks=True) (destination / "sparkring-component-manifest.json").write_text( - json.dumps(lock["cuda"]["components"], indent=2, sort_keys=True) + "\n" + json.dumps(lock["cuda"]["components"], indent=2, sort_keys=True) + "\n", + encoding="utf-8", newline="\n", ) @@ -145,7 +146,7 @@ def apply( "target_head_quantization": False, } receipt.parent.mkdir(parents=True, exist_ok=True) - receipt.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n") + receipt.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n") return receipt diff --git a/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py b/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py index ad34d81a..fad54c08 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py +++ b/runtime/glm53-spark-mtp3-mesh/compute/prepare_compute_source.py @@ -12,7 +12,7 @@ from pathlib import Path HERE = Path(__file__).resolve().parent -LOCK = json.loads((HERE / "source-lock.json").read_text()) +LOCK = json.loads((HERE / "source-lock.json").read_text(encoding="utf-8")) def _sha256(path: Path) -> str: @@ -172,7 +172,7 @@ def prepare(destination: Path, cache: Path | None = None) -> Path: "cuda_archives": cuda_archives, } manifest = destination / "prepared-manifest.json" - manifest.write_text(json.dumps(prepared, indent=2, sort_keys=True) + "\n") + manifest.write_text(json.dumps(prepared, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n") return manifest diff --git a/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py b/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py index 8ec201c6..78b7b7e4 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py +++ b/runtime/glm53-spark-mtp3-mesh/compute/test_compute.py @@ -1,4 +1,5 @@ import hashlib +import ast import io import importlib.util import json @@ -23,6 +24,30 @@ def _module(name: str): verify_compute = _module("verify_compute") +def test_compute_json_io_declares_utf8(): + for name in ("apply_compute.py", "prepare_compute_source.py", "verify_compute.py"): + tree = ast.parse((HERE / name).read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr in ("read_text", "write_text"): + values = {arg.arg: arg.value for arg in node.keywords} + assert ast.literal_eval(values["encoding"]) == "utf-8", (name, node.lineno) + if node.func.attr == "write_text": + assert ast.literal_eval(values["newline"]) == "\n", (name, node.lineno) + + +@pytest.mark.parametrize("relative", ["../outside.py", "/tmp/outside.py", "vllm/../../outside.py", "b12x/not-vllm.py", "vllm\\outside.py"]) +def test_verify_rejects_uncontained_override_paths(tmp_path, relative): + lock = tmp_path / "source-lock.json" + lock.write_text(json.dumps({"vllm": {"files": [[relative, "base", "result"]]}}), encoding="utf-8") + receipt = tmp_path / "receipt.json" + receipt.write_text(json.dumps({ + "source_lock_sha256": hashlib.sha256(lock.read_bytes()).hexdigest(), + "vllm_overrides": {relative: "result"}, + }), encoding="utf-8") + with pytest.raises(ValueError, match="Unsafe vLLM override path"): + verify_compute.verify(tmp_path, receipt, lock) + + @pytest.mark.parametrize("fault", [None, "base", "result", "archive", "extra", "duplicate"]) def test_b12x_selector_overrides_fail_closed(tmp_path: Path, fault: str | None) -> None: root = tmp_path / "source" diff --git a/runtime/glm53-spark-mtp3-mesh/compute/verify_compute.py b/runtime/glm53-spark-mtp3-mesh/compute/verify_compute.py index d2a48b31..5435a01a 100644 --- a/runtime/glm53-spark-mtp3-mesh/compute/verify_compute.py +++ b/runtime/glm53-spark-mtp3-mesh/compute/verify_compute.py @@ -5,7 +5,7 @@ import argparse import hashlib import json -from pathlib import Path +from pathlib import Path, PurePosixPath def _sha256(path: Path) -> str: @@ -18,8 +18,8 @@ def _map_sha256(files: dict[str, str]) -> str: def verify(site_packages: Path, receipt: Path, source_lock: Path) -> dict: - lock = json.loads(source_lock.read_text()) - installed = json.loads(receipt.read_text()) + lock = json.loads(source_lock.read_text(encoding="utf-8")) + installed = json.loads(receipt.read_text(encoding="utf-8")) lock_hash = _sha256(source_lock) if installed["source_lock_sha256"] != lock_hash: raise ValueError("installed receipt uses a different compute source lock") @@ -27,7 +27,15 @@ def verify(site_packages: Path, receipt: Path, source_lock: Path) -> dict: if installed["vllm_overrides"] != expected_vllm: raise ValueError("installed receipt omits or changes vLLM overrides") for relative, expected in expected_vllm.items(): - actual = _sha256(site_packages / relative) + path = PurePosixPath(relative) + target = site_packages / path + if (path.is_absolute() or str(path) != relative or ".." in path.parts + or "\\" in relative or not relative.startswith("vllm/") + or not target.resolve().is_relative_to((site_packages / "vllm").resolve()) + or any((site_packages / Path(*path.parts[:index])).is_symlink() + for index in range(1, len(path.parts) + 1))): + raise ValueError(f"Unsafe vLLM override path: {relative}") + actual = _sha256(target) if actual != expected: raise ValueError(f"installed vLLM hash mismatch for {relative}: {actual}") expected_b12x = installed.get("b12x_files")