diff --git a/performance/harnesses/vllm/prefill_checks/.gitattributes b/performance/harnesses/vllm/prefill_checks/.gitattributes new file mode 100644 index 00000000..d9bd16b0 --- /dev/null +++ b/performance/harnesses/vllm/prefill_checks/.gitattributes @@ -0,0 +1 @@ +*.py text eol=lf diff --git a/performance/harnesses/vllm/prefill_checks/continuation_serve_checks.py b/performance/harnesses/vllm/prefill_checks/continuation_serve_checks.py new file mode 100644 index 00000000..b2245e2f --- /dev/null +++ b/performance/harnesses/vllm/prefill_checks/continuation_serve_checks.py @@ -0,0 +1,249 @@ +"""Bounded semantic and cold-prefill checks, gated on completed API warmup.""" + +import argparse +import os +import json +from pathlib import Path +import re +import subprocess +import time +import urllib.request +import uuid + +BASE = os.environ.get("BENCH_API_BASE", "http://127.0.0.1:8000") +MODEL = os.environ.get("BENCH_MODEL", "glm-5.3-flash-spark") +CONTAINER = os.environ.get("BENCH_RANK0_CONTAINER", "sparkring-model-r0") +if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", CONTAINER): + raise ValueError("Unexpected container name") + + +def cached_prompt_tokens(usage): + """Missing accounting cannot establish cold work or successful cache reuse.""" + details = usage.get("prompt_tokens_details") + cached = details.get("cached_tokens") if isinstance(details, dict) else None + if type(cached) is not int or cached < 0: + raise ValueError("Usage must provide a nonnegative integer cached_tokens") + return cached + + +def get(path): + with urllib.request.urlopen(BASE + path, timeout=10) as r: + return r.read().decode() + + +def post(path, body): + req = urllib.request.Request( + BASE + path, + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=180) as r: + return json.load(r) + + +def ready(): + status = subprocess.check_output( + [ + "ssh", + os.environ.get("BENCH_RANK0_SSH", "rank0"), + "docker inspect --format '{{.State.Health.Status}}' " + CONTAINER, + ], + text=True, + ).strip() + if status != "healthy": + return False + metrics = get("/metrics") + values = { + key: [ + float(line.rsplit(" ", 1)[-1]) + for line in metrics.splitlines() + if line.startswith("vllm:" + key + "{") + or line.startswith("vllm:" + key + " ") + ] + for key in ("num_requests_running", "num_requests_waiting") + } + return all(rows and sum(rows) == 0 for rows in values.values()) + + +def calibrate(tokens, fact): + prefix = ( + "Test " + + uuid.uuid4().hex + + ". The project code is " + + fact + + ". Remember it.\n" + ) + suffix = "\nWhat is the project code? Reply with just the code." + words = tokens + for _ in range(12): + text = ( + prefix + + " ".join( + (["alpha", "beta", "gamma", "delta"] * ((words + 3) // 4))[:words] + ) + + suffix + ) + msg = [{"role": "user", "content": text}] + count = post( + "/tokenize", + {"model": MODEL, "messages": msg, "add_generation_prompt": True}, + )["count"] + if count == tokens: + return msg + words += tokens - count + raise ValueError("Exact prompt calibration failed") + + +def semantic(tokens, fact): + messages = calibrate(tokens, fact) + rows = [] + for phase in ("fresh", "repeated", "extended"): + msg = ( + messages + if phase != "extended" + else [ + { + "role": "user", + "content": messages[0]["content"] + + "\nFinal instruction: give exactly the project code.", + } + ] + ) + started = time.monotonic() + response = post( + "/v1/chat/completions", + { + "model": MODEL, + "messages": msg, + "max_tokens": 384, + "temperature": 0, + "top_p": 1, + }, + ) + choice = response["choices"][0] + answer = (choice["message"].get("content") or "").strip() + usage = response["usage"] + cached = cached_prompt_tokens(usage) + row = { + "tokens": tokens, + "phase": phase, + "seconds": time.monotonic() - started, + "response": response, + "answer_ok": answer == fact and choice["finish_reason"] == "stop", + "cache_ok": cached == 0 if phase == "fresh" else cached > 0, + } + row["pass"] = row["answer_ok"] and row["cache_ok"] + if phase == "fresh": + row["pass"] = row["pass"] and usage["prompt_tokens"] == tokens + rows.append(row) + print( + json.dumps( + {k: v for k, v in row.items() if k != "response"} + | {"answer": answer, "cached_tokens": cached} + ), + flush=True, + ) + yield row + if not row["pass"]: + raise RuntimeError("Semantic/cache check failed; inspect response") + + +def prefill(tokens): + msg = calibrate(tokens, "STONE-7482") + body = { + "model": MODEL, + "messages": msg, + "max_tokens": 1, + "temperature": 0, + "stream": True, + "stream_options": {"include_usage": True}, + } + request = urllib.request.Request( + BASE + "/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + ) + started = time.monotonic() + first = None + usage = None + with urllib.request.urlopen(request, timeout=180) as response: + for line in response: + if not line.startswith(b"data: "): + continue + raw = line[6:].strip() + if raw == b"[DONE]": + break + chunk = json.loads(raw) + if chunk.get("usage"): + usage = chunk["usage"] + for choice in chunk.get("choices", []): + delta = choice.get("delta", {}) + if first is None and any( + delta.get(k) for k in ("content", "reasoning", "reasoning_content") + ): + first = time.monotonic() - started + if first is None or usage is None: + raise RuntimeError("No streamed token or final usage") + assert usage["prompt_tokens"] == tokens + assert cached_prompt_tokens(usage) == 0 + return { + "tokens": tokens, + "ttft_seconds": first, + "tokens_per_second": tokens / first, + "usage": usage, + } + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("phase", choices=["semantic", "prefill"]) + p.add_argument("--output", type=Path, required=True) + p.add_argument("--sizes", default="16384,10240,32768,65536,8192") + p.add_argument("--samples", type=int, default=3) + args = p.parse_args() + assert not args.output.exists() + deadline = time.monotonic() + 1200 + announced = False + while not ready(): + if time.monotonic() > deadline: + raise TimeoutError("Warmup/idle readiness deadline") + if not announced: + print( + "Waiting for completed warmup and zero running/waiting requests.", + flush=True, + ) + announced = True + time.sleep(5) + print("Warmup complete; server idle. Starting controlled checks.", flush=True) + rows = [] + + def save(): + args.output.write_text( + json.dumps({"phase": args.phase, "rows": rows}, indent=2), encoding="utf-8" + ) + + sizes = list(map(int, args.sizes.split(","))) + if args.phase == "semantic": + for i, tokens in enumerate(sizes): + for row in semantic(tokens, "RIVER-" + str(5938 + i)): + rows.append(row) + save() + else: + for phase in ("warm", "measured"): + for sample in range(1 if phase == "warm" else args.samples): + for tokens in sizes: + if not ready(): + raise RuntimeError( + "Concurrent activity detected before timing sample" + ) + row = prefill(tokens) | {"phase": phase, "sample": sample} + rows.append(row) + save() + print( + json.dumps({k: v for k, v in row.items() if k != "usage"}), + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/performance/harnesses/vllm/prefill_checks/test_prefill_checks.py b/performance/harnesses/vllm/prefill_checks/test_prefill_checks.py new file mode 100644 index 00000000..1cf661c2 --- /dev/null +++ b/performance/harnesses/vllm/prefill_checks/test_prefill_checks.py @@ -0,0 +1,137 @@ +"""Exercise request timing and readiness gates without contacting a server.""" + +import importlib.util +import io +import json +from pathlib import Path + +import pytest + +HERE = Path(__file__).parent + + +@pytest.fixture( + params=sorted( + path for path in HERE.glob("*_checks.py") if not path.name.startswith("test_") + ) +) +def harness(request): + spec = importlib.util.spec_from_file_location( + "prefill_harness_" + request.param.stem, request.param + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + "health,running,waiting,expected", + [ + ("healthy", 0, 0, True), + ("starting", 0, 0, False), + ("healthy", 1, 0, False), + ("healthy", 0, 1, False), + ], +) +def test_readiness_requires_healthy_idle_service( + harness, monkeypatch, health, running, waiting, expected +): + monkeypatch.setattr(harness.subprocess, "check_output", lambda *a, **kw: health) + monkeypatch.setattr( + harness, + "get", + lambda path: ( + f'vllm:num_requests_running{{model="m"}} {running}\n' + f'vllm:num_requests_waiting{{model="m"}} {waiting}\n' + ), + ) + assert harness.ready() is expected + + +def test_missing_metrics_do_not_count_as_zero(harness, monkeypatch): + monkeypatch.setattr(harness.subprocess, "check_output", lambda *a, **kw: "healthy") + monkeypatch.setattr(harness, "get", lambda path: "") + assert harness.ready() is False + + +@pytest.mark.parametrize( + "tokens,cached,rejected", [(8192, 0, False), (8191, 0, True), (8192, 1, True)] +) +def test_timing_rejects_wrong_prompt_length_or_cached_work( + harness, monkeypatch, tokens, cached, rejected +): + clock = iter([10.0, 10.25]) + clock_name = ( + "perf_counter" + if harness.__name__.endswith("mhc_precise_checks") + else "monotonic" + ) + monkeypatch.setattr(harness.time, clock_name, lambda: next(clock)) + monkeypatch.setattr( + harness, "calibrate", lambda *args: [{"role": "user", "content": "test"}] + ) + chunks = [ + {"choices": [{"delta": {"content": ""}}]}, + {"choices": [{"delta": {"content": "answer"}}]}, + { + "choices": [], + "usage": { + "prompt_tokens": tokens, + "prompt_tokens_details": {"cached_tokens": cached}, + }, + }, + ] + stream = ( + b"".join(b"data: " + json.dumps(chunk).encode() + b"\n" for chunk in chunks) + + b"data: [DONE]\n" + ) + monkeypatch.setattr( + harness.urllib.request, "urlopen", lambda *a, **kw: io.BytesIO(stream) + ) + if rejected: + with pytest.raises(AssertionError): + harness.prefill(8192) + else: + result = harness.prefill(8192) + assert result["ttft_seconds"] == 0.25 + assert result["tokens_per_second"] == 32768 + + +@pytest.mark.parametrize( + "details", + [ + "omitted", + None, + {}, + {"cached_tokens": False}, + {"cached_tokens": "0"}, + {"cached_tokens": -1}, + ], +) +def test_timing_rejects_unproven_cache_accounting(harness, monkeypatch, details): + clock = iter([10.0, 10.25]) + clock_name = ( + "perf_counter" + if harness.__name__.endswith("mhc_precise_checks") + else "monotonic" + ) + monkeypatch.setattr(harness.time, clock_name, lambda: next(clock)) + monkeypatch.setattr( + harness, "calibrate", lambda *args: [{"role": "user", "content": "test"}] + ) + usage = {"prompt_tokens": 8192} + if details != "omitted": + usage["prompt_tokens_details"] = details + chunks = [ + {"choices": [{"delta": {"content": "answer"}}]}, + {"choices": [], "usage": usage}, + ] + stream = ( + b"".join(b"data: " + json.dumps(chunk).encode() + b"\n" for chunk in chunks) + + b"data: [DONE]\n" + ) + monkeypatch.setattr( + harness.urllib.request, "urlopen", lambda *a, **kw: io.BytesIO(stream) + ) + with pytest.raises(ValueError, match="cached_tokens"): + harness.prefill(8192) diff --git a/performance/records/glm53-flash/continuation-checkpoints-20260906/.gitattributes b/performance/records/glm53-flash/continuation-checkpoints-20260906/.gitattributes new file mode 100644 index 00000000..6ea0cf98 --- /dev/null +++ b/performance/records/glm53-flash/continuation-checkpoints-20260906/.gitattributes @@ -0,0 +1 @@ +*.json -text whitespace=cr-at-eol diff --git a/performance/records/glm53-flash/continuation-checkpoints-20260906/README.md b/performance/records/glm53-flash/continuation-checkpoints-20260906/README.md new file mode 100644 index 00000000..80921a06 --- /dev/null +++ b/performance/records/glm53-flash/continuation-checkpoints-20260906/README.md @@ -0,0 +1,98 @@ +# Fixed-8K continuation checkpoint evidence + +Status: **research-only**. The recorded runtime completes cold long prompts +without splitting the final continuation solely to export recurrent checkpoints. +This record describes the exact image below; it does not qualify the rebuilt +source composition that also adds request attribution. + +## Conditions + +Four DGX Spark GB10 GPUs, TP4/DCP4/PP1, native MTP3, 8,192 maximum batched +tokens, 16 sequences, and 24 GiB KV cache per rank. The model is +`local-inference-lab/GLM-5.3-Flash-NVFP4-Spark` at revision +`df116c4fb16b1d37ae43d2cfd624de26ffbc832e`. The image config is +`sha256:489d1975619e9083d14f14bcd1c6cbb4a96c41e3ab3978af048dc3ed2bb452a8`. +Both `SPARK_GDN_PREFILL_CHECKPOINTS=2` and +`SPARK_GDN_CONTINUATION_CHECKPOINTS=1` are enabled. The native virtual mesh, +graph-only decode geometry, linked posting, B12X compute patches, and SparkCache +remain in the composition; periodic full capture is off. + +Requests have concurrency one, temperature zero, one output token, and a unique +prefix. Exact prompt usage and zero cached tokens are checked for every sample. +There is no restarted paired control. Four scheduler/allocator/helper files +changed, with a separate cache root and regenerated ownership/image inventories. +Component probes used a source-identical child before a metadata-only +deterministic rebuild. Full conditions are in [the record](continuation-evidence.json). + +## Measurement + +TTFT spans submission through the first nonempty streamed content or reasoning +delta, including client/network overhead. The historical Windows harness uses +`time.monotonic` backed by GetTickCount64, with about 15.625 ms granularity. +Reported decimal precision is not clock accuracy. Startup/API warmup and idle +gauges passed; one shape-warmup row per 8K–64K size is excluded. Prior semantic +and prefill requests warmed 128K. Each size has three measured samples. + +The table uses prompt tokens divided by median TTFT; min/max describe those +three TTFTs. Raw rows are [8K–64K](continuation-prefill.json) and +[128K](continuation-prefill-128k.json). [Provenance](provenance.json) identifies +the retained historical harness and the portable public copy; those hashes are +not signed execution-time attestations. + +## Result + +| Prompt tokens | Median TTFT | Min–max TTFT | Prompt tok/s | +|---:|---:|---:|---:| +| 8,192 | 2.797 s | 2.782–2.813 s | 2,929 | +| 16,384 | 5.672 s | 5.656–5.672 s | 2,889 | +| 32,768 | 11.328 s | 11.297–11.344 s | 2,893 | +| 65,536 | 22.719 s | 22.718–22.750 s | 2,885 | +| 131,072 | 45.875 s | 45.782–45.875 s | 2,857 | + +Six [recurrence cases](continuation-recurrence.json) and three +[convolution cases](continuation-convolution.json) passed on GPU. The recorded +model checks include 15 regular semantic/cache cases, four concurrent cold +requests, and 25 exact answers across long-context attempts. All four workers +recorded an 8,192→16,384 continuation with checkpoint ends 14,336 and 15,360 +across 34 GDN layers. + +## Conclusion + +The fixed-8K continuation implementation completed these cases at about +2,857–2,929 prompt tokens/s. These candidate-only observations establish no +isolated percentage speedup. The scheduler keeps the 8K chunk ceiling; it +exports intermediate state within an eligible final chunk rather than adding +checkpoint-driven model passes. + +## Limitations + +Three samples and synthetic fact retrieval do not establish general quality, +decode performance, concurrency performance, arbitrary-length correctness, or +other chunk sizes. CPU tests exercise a 6K schedule, but only fixed 8K has this +serving evidence. Historical screenshots used different harnesses/compositions. + +Two 128K cache-reuse expectations missed and recomputed safely, including one +after worker publication. Later repeated/extended checks reused 129,024 tokens. +Scheduler visibility was not traced; this is not attributed as a continuation +regression. The rank-three launch memory gate passed on recheck without a reboot +or threshold change. The attribution-containing rebuild requires its own GPU, +startup, and serving validation. + +## Reproduction + +The [portable harness](../../../harnesses/vllm/prefill_checks/continuation_serve_checks.py) +retains the historical timing and request logic and additionally rejects +missing or malformed cache-token accounting. Every retained timing row contains +explicit integer cache accounting, so this check does not change the record. +Set `BENCH_API_BASE`, +`BENCH_MODEL`, `BENCH_RANK0_SSH`, and `BENCH_RANK0_CONTAINER` for an explicitly +authorized idle deployment, then run: + +```bash +python performance/harnesses/vllm/prefill_checks/continuation_serve_checks.py prefill \ + --sizes 8192,16384,32768,65536,131072 --samples 3 --output /path/to/absent-results.json +``` + +This command sends inference requests and reads rank-zero Docker health over +SSH. Establish all-rank startup/health separately. The published harness has +offline gate coverage but has not been rerun on a GPU. diff --git a/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-convolution.json b/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-convolution.json new file mode 100644 index 00000000..1b81873f --- /dev/null +++ b/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-convolution.json @@ -0,0 +1,29 @@ +{ + "cases": [ + { + "tail": 2048, + "checkpoint_offsets": [ + 1024, + 0 + ], + "pass": true + }, + { + "tail": 4096, + "checkpoint_offsets": [ + 2048, + 3072 + ], + "pass": true + }, + { + "tail": 8192, + "checkpoint_offsets": [ + 6144, + 7168 + ], + "pass": true + } + ], + "pass": true +} \ No newline at end of file diff --git a/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-evidence.json b/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-evidence.json new file mode 100644 index 00000000..cab803db --- /dev/null +++ b/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-evidence.json @@ -0,0 +1,94 @@ +{ + "Conditions": { + "hardware": "Four NVIDIA DGX Spark GB10 GPUs, 48 SMs each, virtual mesh over the existing physical ring", + "model": "local-inference-lab/GLM-5.3-Flash-NVFP4-Spark", + "model_revision": "df116c4fb16b1d37ae43d2cfd624de26ffbc832e", + "tensor_parallel": 4, + "decode_context_parallel": 4, + "pipeline_parallel": 1, + "mtp_steps": 3, + "max_num_batched_tokens": 8192, + "max_num_seqs": 16, + "kv_cache_gib_per_rank": 24, + "concurrency": 1, + "max_output_tokens": 1, + "temperature": 0, + "exact_prompt_tokens": true, + "cache_state": "Unique prompt prefix for every timing request; cached_tokens=0 required for each sample; periodic full captures off.", + "retained_features": "Native virtual mesh, graph-only decode geometry, linked posting, existing B12X compute patches, SparkCache; no dynamic pressure-aware chunk sizing.", + "image": "sha256:489d1975619e9083d14f14bcd1c6cbb4a96c41e3ab3978af048dc3ed2bb452a8", + "flag": "SPARK_GDN_CONTINUATION_CHECKPOINTS=1", + "comparison": "Single candidate, no restarted paired control. Scheduler/allocator/helper extension only; four runtime sources plus ownership/inventory changed; fresh cache root. Component probes used source-identical61f5817f child before deterministic metadata-only rebuild." + }, + "Measurement": { + "metric": "Seconds from request submission to first nonempty streamed content/reasoning delta, including client/network overhead; prompt tokens / TTFT.", + "clock": "Windows Python time.monotonic; GetTickCount64, approximately15.625ms granularity. Do not treat reported decimal precision as clock accuracy.", + "warmup": "Startup/API warmup complete and running/waiting gauges zero. One warm row per8/16/32/64K shape excluded;128K already warmed by preceding semantic/full-prefill calls.", + "samples_per_shape": 3, + "raw_files": [ + "continuation-prefill.json", + "continuation-prefill-128k.json" + ], + "aggregation": "Median TTFT, min/max of three measured TTFTs; tokens divided by median TTFT." + }, + "Result": [ + { + "tokens": 8192, + "samples": 3, + "median_ttft_seconds": 2.7969999999913853, + "prefill_tokens_per_second": 2928.8523418038008, + "min_ttft": 2.7819999999919673, + "max_ttft": 2.812999999994645 + }, + { + "tokens": 16384, + "samples": 3, + "median_ttft_seconds": 5.671999999991385, + "prefill_tokens_per_second": 2888.5754583964886, + "min_ttft": 5.6560000000026776, + "max_ttft": 5.672000000005937 + }, + { + "tokens": 32768, + "samples": 3, + "median_ttft_seconds": 11.328000000008615, + "prefill_tokens_per_second": 2892.6553672294385, + "min_ttft": 11.296999999991385, + "max_ttft": 11.343999999997322 + }, + { + "tokens": 65536, + "samples": 3, + "median_ttft_seconds": 22.718999999997322, + "prefill_tokens_per_second": 2884.6340067788074, + "min_ttft": 22.718000000008033, + "max_ttft": 22.75 + }, + { + "tokens": 131072, + "samples": 3, + "median_ttft_seconds": 45.875, + "prefill_tokens_per_second": 2857.1553133514985, + "min_ttft": 45.78199999999197, + "max_ttft": 45.875 + } + ], + "Conclusion": "Fixed8192 continuation completed the stated cases with roughly2857\u20132929 prompt tokens/s at measured8K\u2013128K sizes. No isolated percentage speedup is established by these observations.", + "Limitations": [ + "Three measurements per shape, synthetic fact-retrieval prompts and1 output token; no comprehensive quality, decode, concurrency or arbitrary-length claim.", + "Historical screenshots used different harness/compositions and are not matched controls.", + "Two128K reuse expectations missed and recomputed safely, including one after worker publication; later full repeated/extended checks reused129024 tokens. Scheduler visibility was not traced, so no attribution as a new continuation regression.", + "Rank3 launch memory gate passed on recheck without reboot or threshold change; timestamp-normalized image rebuild recorded separately." + ], + "Correctness": { + "gpu_recurrence_cases": 6, + "gpu_convolution_cases": 3, + "regular_semantic_cache_cases": 15, + "concurrent_cold_requests": 4, + "total_exact_answers_across_long_context_attempts": 25, + "cache_reuse_expectation_misses_128k": 2, + "later_128k_reuse_tokens": 129024, + "activation": "All4 workers recorded8192->16384 with checkpoint ends14336/15360 across34GDNlayers.", + "universal_correctness_claim": false + } +} diff --git a/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-prefill-128k.json b/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-prefill-128k.json new file mode 100644 index 00000000..3ecf7993 --- /dev/null +++ b/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-prefill-128k.json @@ -0,0 +1,62 @@ +{ + "prior_warmup": "completed 128K semantic and repeated full-prefill requests", + "rows": [ + { + "tokens": 131072, + "ttft_seconds": 45.78199999999197, + "tokens_per_second": 2862.959241623848, + "usage": { + "prompt_tokens": 131072, + "total_tokens": 131073, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 130048 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 0 + }, + { + "tokens": 131072, + "ttft_seconds": 45.875, + "tokens_per_second": 2857.1553133514985, + "usage": { + "prompt_tokens": 131072, + "total_tokens": 131073, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 130048 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 1 + }, + { + "tokens": 131072, + "ttft_seconds": 45.875, + "tokens_per_second": 2857.1553133514985, + "usage": { + "prompt_tokens": 131072, + "total_tokens": 131073, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 130048 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 2 + } + ] +} \ No newline at end of file diff --git a/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-prefill.json b/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-prefill.json new file mode 100644 index 00000000..930519cf --- /dev/null +++ b/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-prefill.json @@ -0,0 +1,309 @@ +{ + "phase": "prefill", + "rows": [ + { + "tokens": 8192, + "ttft_seconds": 2.7969999999913853, + "tokens_per_second": 2928.8523418038008, + "usage": { + "prompt_tokens": 8192, + "total_tokens": 8193, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 7168 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "warm", + "sample": 0 + }, + { + "tokens": 16384, + "ttft_seconds": 5.625, + "tokens_per_second": 2912.711111111111, + "usage": { + "prompt_tokens": 16384, + "total_tokens": 16385, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 15360 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "warm", + "sample": 0 + }, + { + "tokens": 32768, + "ttft_seconds": 11.297000000005937, + "tokens_per_second": 2900.593077806743, + "usage": { + "prompt_tokens": 32768, + "total_tokens": 32769, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 31744 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "warm", + "sample": 0 + }, + { + "tokens": 65536, + "ttft_seconds": 22.702999999994063, + "tokens_per_second": 2886.66696031437, + "usage": { + "prompt_tokens": 65536, + "total_tokens": 65537, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 64512 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "warm", + "sample": 0 + }, + { + "tokens": 8192, + "ttft_seconds": 2.7819999999919673, + "tokens_per_second": 2944.6441409143254, + "usage": { + "prompt_tokens": 8192, + "total_tokens": 8193, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 7168 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 0 + }, + { + "tokens": 16384, + "ttft_seconds": 5.6560000000026776, + "tokens_per_second": 2896.746817537525, + "usage": { + "prompt_tokens": 16384, + "total_tokens": 16385, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 15360 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 0 + }, + { + "tokens": 32768, + "ttft_seconds": 11.296999999991385, + "tokens_per_second": 2900.5930778104794, + "usage": { + "prompt_tokens": 32768, + "total_tokens": 32769, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 31744 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 0 + }, + { + "tokens": 65536, + "ttft_seconds": 22.718999999997322, + "tokens_per_second": 2884.6340067788074, + "usage": { + "prompt_tokens": 65536, + "total_tokens": 65537, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 64512 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 0 + }, + { + "tokens": 8192, + "ttft_seconds": 2.812999999994645, + "tokens_per_second": 2912.1933878477053, + "usage": { + "prompt_tokens": 8192, + "total_tokens": 8193, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 7168 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 1 + }, + { + "tokens": 16384, + "ttft_seconds": 5.672000000005937, + "tokens_per_second": 2888.575458389078, + "usage": { + "prompt_tokens": 16384, + "total_tokens": 16385, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 15360 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 1 + }, + { + "tokens": 32768, + "ttft_seconds": 11.328000000008615, + "tokens_per_second": 2892.6553672294385, + "usage": { + "prompt_tokens": 32768, + "total_tokens": 32769, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 31744 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 1 + }, + { + "tokens": 65536, + "ttft_seconds": 22.718000000008033, + "tokens_per_second": 2884.7609824798324, + "usage": { + "prompt_tokens": 65536, + "total_tokens": 65537, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 64512 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 1 + }, + { + "tokens": 8192, + "ttft_seconds": 2.7969999999913853, + "tokens_per_second": 2928.8523418038008, + "usage": { + "prompt_tokens": 8192, + "total_tokens": 8193, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 7168 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 2 + }, + { + "tokens": 16384, + "ttft_seconds": 5.671999999991385, + "tokens_per_second": 2888.5754583964886, + "usage": { + "prompt_tokens": 16384, + "total_tokens": 16385, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 15360 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 2 + }, + { + "tokens": 32768, + "ttft_seconds": 11.343999999997322, + "tokens_per_second": 2888.5754583927833, + "usage": { + "prompt_tokens": 32768, + "total_tokens": 32769, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 31744 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 2 + }, + { + "tokens": 65536, + "ttft_seconds": 22.75, + "tokens_per_second": 2880.703296703297, + "usage": { + "prompt_tokens": 65536, + "total_tokens": 65537, + "completion_tokens": 1, + "prompt_tokens_details": { + "cached_tokens": 0, + "created_cache_tokens": 64512 + }, + "completion_tokens_details": { + "reasoning_tokens": 1 + } + }, + "phase": "measured", + "sample": 2 + } + ] +} \ No newline at end of file diff --git a/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-recurrence.json b/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-recurrence.json new file mode 100644 index 00000000..64a43b68 --- /dev/null +++ b/performance/records/glm53-flash/continuation-checkpoints-20260906/continuation-recurrence.json @@ -0,0 +1,47 @@ +{ + "gpu_executed": true, + "kernel_source_pins": { + "b12x.sequence.kda_prefill._impl": "0d0fc64b5fddd269d823a57f14c09a9e0a1b38a4f802c39df07353f34873a270", + "b12x.sequence.kda_prefill._cute_kernels": "df12ee84e1677ce1a200d1aab488c9ce5ae774d21f70b7f15b39c636623ab753", + "b12x.sequence.kda_prefill._policy": "ec64f07f841675501ff279bae3074d374ca8015a982ab12dc837ea2d03720a8e", + "b12x.sequence.kda_prefill.metadata": "7a9ef9ef3c59c88e4816bea6c9a74ebadd91a85632b42e221a5dd4009c5bb65c" + }, + "cases": [ + { + "tail_tokens": 2048, + "inplace": false, + "pass": true + }, + { + "tail_tokens": 2048, + "inplace": true, + "pass": true + }, + { + "tail_tokens": 4096, + "inplace": false, + "pass": true + }, + { + "tail_tokens": 4096, + "inplace": true, + "pass": true + }, + { + "tail_tokens": 8192, + "inplace": false, + "pass": true + }, + { + "tail_tokens": 8192, + "inplace": true, + "pass": true + } + ], + "pass": true, + "limitations": [ + "no convolution check", + "no scheduler/refcount integration", + "no full-model correctness or speed result" + ] +} \ No newline at end of file diff --git a/performance/records/glm53-flash/continuation-checkpoints-20260906/provenance.json b/performance/records/glm53-flash/continuation-checkpoints-20260906/provenance.json new file mode 100644 index 00000000..1e677bf1 --- /dev/null +++ b/performance/records/glm53-flash/continuation-checkpoints-20260906/provenance.json @@ -0,0 +1,28 @@ +[ + { + "public_file": "continuation-prefill.json", + "sha256": "78a35a51a0b851cba7ad56307e54a2316b8ce2e3ce69a2339919b7d575f5e128", + "transformation": "None: exact source bytes; private source location omitted." + }, + { + "public_file": "continuation-prefill-128k.json", + "sha256": "3ba7cb58ea757905e4e164e390f6409418bf04d5451146190b68fb2dcce9a251", + "transformation": "None: exact source bytes; private source location omitted." + }, + { + "public_file": "continuation-recurrence.json", + "sha256": "5ff46ae9c343e889d7319b7f0db3fafe28bea79762c1571ca64136974ed3cdbf", + "transformation": "None: exact source bytes; private source location omitted." + }, + { + "public_file": "continuation-convolution.json", + "sha256": "42ff4bbf70ce2d649a95af219447b3c5a262539bca8dece06205a7c3ab2dbc36", + "transformation": "None: exact source bytes; private source location omitted." + }, + { + "public_file": "../../../harnesses/vllm/prefill_checks/continuation_serve_checks.py", + "historical_harness_sha256": "f8c8d0f75b13ed833371bbef89c4e47a89ec545dfa39135df4969f39ec0f4f05", + "public_harness_sha256": "8b7b1cb03a1967317e3d02f21a1a6dbb5b89e29a434a1dff5f7069d50175e962", + "transformation": "Endpoint/model/SSH alias and container selection parameterized; os import added where required, unused import removed, UTF-8 output made explicit, source formatted. Timing/request logic retained; missing or malformed cache-token accounting explicitly rejected. All retained timing rows contain explicit integer cached_tokens=0. Public copy has not been rerun on GPU. Historical source digest was computed from a retained copy, not signed at execution time." + } +] diff --git a/runtime/glm53-spark-mtp3-mesh/performance/Dockerfile b/runtime/glm53-spark-mtp3-mesh/performance/Dockerfile index d248198b..6add0358 100644 --- a/runtime/glm53-spark-mtp3-mesh/performance/Dockerfile +++ b/runtime/glm53-spark-mtp3-mesh/performance/Dockerfile @@ -1,10 +1,10 @@ ARG PARENT_IMAGE=ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:67dc0ae453baaae6831ccec1d259b4ef8b236a8b0dc9f747d901b95c66ec1987 FROM ${PARENT_IMAGE} RUN --mount=type=bind,target=/performance python3 -S -B /performance/install.py -ENV SPARK_GDN_PREFILL_CHECKPOINTS=2 +ENV SPARK_GDN_PREFILL_CHECKPOINTS=2 SPARK_GDN_CONTINUATION_CHECKPOINTS=1 LABEL org.sparkring.runtime.status="research-only" \ org.sparkring.runtime.profile="glm53-mtp3-cache-checkpoints" \ - org.sparkcache.commit="48bbd2be4a7b972e56632a2d7b934bac5460f272" \ + org.sparkcache.commit="b5aca7cd3d3f7e7a14636bf6e5fa1f50a9650168" \ org.sparkcache.cuda-placement-sha256="2657cdd2e54a097c9544e4c79ae62c0646db6db123ff24e4f0c384238c3a1e8d" \ org.sparkring.sircl.manifest-sha256="c0fd5567442b08b908cc193f36d0864e262573c7e5d232509479a823cface742" ENTRYPOINT ["python3", "-S", "-B", "/opt/sparkring/bin/start-performance.py"] diff --git a/runtime/glm53-spark-mtp3-mesh/performance/README.md b/runtime/glm53-spark-mtp3-mesh/performance/README.md index 4c297610..585ada4b 100644 --- a/runtime/glm53-spark-mtp3-mesh/performance/README.md +++ b/runtime/glm53-spark-mtp3-mesh/performance/README.md @@ -5,9 +5,14 @@ The build combines GLM-5.3 native MTP3 compute, verified persistent caching, explicit recurrent checkpoints, and stream-ordered hardware mesh transport. The recipe preserves the parent model weights and does not change host fabric. +This optional research builder uses its own pinned parent and source overlays. +The shared GLM image and its topology profiles are defined separately in +[`runtime/sparkring/source_image`](../../sparkring/source_image/README.md). +This builder does not replace those profiles or their image receipts. + ## Build inputs -Use a clean SparkCache checkout at `48bbd2be4a7b972e56632a2d7b934bac5460f272`. +Use a clean SparkCache checkout at `b5aca7cd3d3f7e7a14636bf6e5fa1f50a9650168`. It contains the merged restore/publication improvements, periodic-capture option, and backlog gauges. Periodic full capture defaults off; enabling it trades more writes for shorter history reconstruction. @@ -129,5 +134,24 @@ module that lacks the rule will not gain it merely from an environment override. This packaging change keeps the existing timeout policy. A workload-aware progress signal remains separate work. CPU -tests demonstrate that frozen output with growing KV occupancy still triggers -the configured timeout; they do not establish a safe universal timeout. +tests distinguish allocation progress from a stalled engine; they do not +establish a safe universal timeout. + +## Request reuse accounting + +The image build includes [scheduler attribution hooks](attribution/README.md) +for the connector's opt-in request ledger. These hooks distinguish admitted +local reuse, finalized persistent restoration, and accepted prompt work across +preemption attempts. They are inactive unless the connector enables request +cache events. Source availability does not change a published image receipt; +a rebuilt image must be validated before serving evidence is claimed. + +## Continuation checkpoint sources + +Source builds preserve the [four continuation checkpoint files](continuation/README.md) +from the source-attested continuation serving image. Both recurrent checkpoint +flags are enabled in the build recipe. The installer validates checkpoint +ownership, applies those four replacements, then applies the matching request +attribution transform and generates the full image inventory. This source build +composition requires separate serving validation; it does not change the +immutable published image contract. diff --git a/runtime/glm53-spark-mtp3-mesh/performance/attribution/README.md b/runtime/glm53-spark-mtp3-mesh/performance/attribution/README.md new file mode 100644 index 00000000..01ddf3da --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/attribution/README.md @@ -0,0 +1,83 @@ +# Request cache attribution at scheduler boundaries + +Status: **implemented** with CPU scheduler-seam coverage. The MTP3 profile +remains **research-only**; rebuilt-image validation is required. + +`patch_scheduler.py` applies a byte-checked transform to either the recurrent +checkpoint scheduler or its explicitly pinned final-chunk continuation variant. The payload files and their manifest +are immutable inputs. The image installer first verifies every checkpoint +ownership dependency, applies the transform, and changes only the scheduler +entry to the transform's expected output hash. The image receipt records both +transform hashes and verifies the installed scheduler file. + +## Connector interface + +Instrumentation is active only when the connector sets +`request_cache_events_enabled = True`, and calls its optional method: + +```python +record_request_cache_event(request, event, **fields) +``` + +SparkCache may enable that property with `SPARK_CONTEXT_CACHE_TRACE_REUSE=1`. +Connectors without the property receive no callbacks or dispatch metadata. +The runtime does not add Prometheus labels or public API fields. The connector +owns bounded request ledgers, aggregate counters, and opt-in request traces. + +Each event includes `preemptions`, copied from `request.num_preemptions`. +Token counts refer to the original prompt and exclude generated tokens. + +| Event | Fields | Authoritative boundary | +|---|---|---| +| `admitted` | `local_tokens`, `external_tokens`, `lease_attached`, `source` | Successful block allocation after local/remote reconciliation; `source` is `gpu_lease` or `prefix_lookup` | +| `restore_finalized` | `success`, `valid_prefix_tokens` | All-worker receive finalization after failure handling and the full-hit sampling-token adjustment | +| `prompt_step_completed` | `start_token`, `end_token`, `stale=False` | Model output that passes invalid-load, abort, stale-output, and attempt-generation checks | +| `preempted` | Common fields only | Request counters reset and preemption generation incremented | +| `finished` | `status` as the request-status enum name | Terminal request cleanup, before the connector's finish hook | + +The admission event describes reuse accepted for allocation, not completed +inference. External reuse remains provisional until receive finalization. +The connector must reconcile a failed restore against `valid_prefix_tokens`; +a failure does not increment the preemption generation. A replacement lookup +can therefore admit a prefix in the same generation. An ordinary resume after +an asynchronous load does not emit another admission event. Lease attribution +survives deferred block allocation and is emitted once allocation succeeds; +its saved generation cannot be reused after preemption. + +A full external prompt restore can materialize every prompt token while the +scheduler recomputes its final token to produce sampling logits. Restored state +span and external prompt tokens reused are separate quantities. A resident GPU +lease is local reuse, even when another request originally populated that +lease through persistent restoration. + +## Completed prompt work + +The scheduler snapshots each dispatched token range before advancing request +counters. Each range is clipped to the original prompt length and retains its +preemption generation. Matching accepted output emits that saved range once; +subsequent scheduling or counter resets cannot change its endpoints. + +A completed decode step can emit an empty prompt range. This allows the +connector to commit pending prefix reuse after a resumed request completes +inference without additional prompt computation. It contributes zero prompt +compute tokens. After that generation has consumed its admitted reuse, ordinary +decode steps do not allocate empty prompt-range entries or emit redundant events. +Stale, failed, or aborted output earns no completed-work +credit. These counters do not measure discarded GPU execution or kernel time. + +Cumulative prompt work and reuse across preemption attempts can exceed the +original prompt length. Do not derive actual computation by subtracting cache +offers from the prompt length. Request completion and missing-event handling +remain connector responsibilities. + +## Offline validation + +```bash +python -m pytest runtime/glm53-spark-mtp3-mesh/performance/attribution -q +``` + +Tests execute transformed scheduler methods and admission/output branches. +They cover partial local tails, lease adoption, prompt clipping, asynchronous +counter advancement, duplicate completion, failed restores, stale output, +preemption, terminal cleanup, and connectors without instrumentation. They do +not qualify CUDA execution, cache restoration, or hardware serving behavior. diff --git a/runtime/glm53-spark-mtp3-mesh/performance/attribution/patch_scheduler.py b/runtime/glm53-spark-mtp3-mesh/performance/attribution/patch_scheduler.py new file mode 100644 index 00000000..fbf552d8 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/attribution/patch_scheduler.py @@ -0,0 +1,142 @@ +"""Instrument source-pinned MTP3 scheduler decisions without changing cache policy.""" + +import ast +import hashlib +from pathlib import Path + +BEFORE_SHA256 = "ce9460834e08f97dbbfeb3f1238b78ee6a3363dd59a2aef8ceeb715857385895" +AFTER_SHA256 = "9500c99fd5f7d82e4c5247c41b64fd4cfc085ca39303db8991f4dcb7e5196a68" +CONTINUATION_BEFORE_SHA256 = "f46c40c1c41daf2bab4566dd185d320f4ec1fb11e8d632c90c47b0f1bb1808fa" +CONTINUATION_AFTER_SHA256 = "6c784cb6d30d89a078e386081f650e7249269704f2bea06c063c656c71bd9cf2" +SOURCE_TRANSFORMS = { + BEFORE_SHA256: AFTER_SHA256, + CONTINUATION_BEFORE_SHA256: CONTINUATION_AFTER_SHA256, +} + + +METHODS = ''' def _sparkcache_record_event(self, request, event, **fields): + connector = self.connector + if not getattr(connector, "request_cache_events_enabled", False): + return + callback = getattr(connector, "record_request_cache_event", None) + if callback is not None: + fields.setdefault("preemptions", request.num_preemptions) + callback(request, event, **fields) + + def _sparkcache_capture_prompt_steps(self, scheduler_output): + if not getattr(self.connector, "request_cache_events_enabled", False): + return + # Preserve dispatch positions: async scheduling advances request counters + # before the matching output arrives, and preemption can reset them. + steps = {} + for req_id, count in scheduler_output.num_scheduled_tokens.items(): + request = self.requests[req_id] + start = min(request.num_computed_tokens, request.num_prompt_tokens) + end = min(request.num_computed_tokens + count, request.num_prompt_tokens) + if end > start or getattr(request, "_sparkcache_consumed_generation", None) != request.num_preemptions: + steps[req_id] = (start, end, request.num_preemptions) + scheduler_output._sparkcache_prompt_steps = steps + + def _sparkcache_complete_prompt_step(self, scheduler_output, request, stale): + step = getattr(scheduler_output, "_sparkcache_prompt_steps", {}).pop( + request.request_id, None + ) + if step is None or stale or step[2] != request.num_preemptions: + return + if step[0] == step[1] and getattr(request, "_sparkcache_consumed_generation", None) == step[2]: + return + self._sparkcache_record_event( + request, "prompt_step_completed", start_token=step[0], + end_token=step[1], preemptions=step[2], stale=False, + ) + request._sparkcache_consumed_generation = step[2] + +''' + +TRANSFORMS = ( + (' def _preempt_request(\n', METHODS + ' def _preempt_request(\n'), + (' local_lease_alternative = None\n', + ''' local_lease_alternative = None + cache_trace_lease_attached = ( + getattr(request, "_sparkcache_pending_lease_generation", None) + == request.num_preemptions + ) +'''), + (' if attached_tokens:\n', + ''' if attached_tokens: + cache_trace_lease_attached = True + # Allocation can defer this attached request; retain + # attribution until admission in the same attempt. + request._sparkcache_pending_lease_generation = request.num_preemptions +'''), + (' # Record at admission so unscheduled lookups are not counted.\n', + ''' if did_prefix_cache_lookup or cache_trace_lease_attached: + cache_trace_local = min( + num_new_local_computed_tokens if did_prefix_cache_lookup + else request.num_computed_tokens, request.num_prompt_tokens + ) + self._sparkcache_record_event( + request, "admitted", local_tokens=cache_trace_local, + external_tokens=min(num_external_computed_tokens, + request.num_prompt_tokens - cache_trace_local), + lease_attached=cache_trace_lease_attached, + source="gpu_lease" if cache_trace_lease_attached else "prefix_lookup", + ) + request._sparkcache_pending_lease_generation = None + request._sparkcache_consumed_generation = None + + # Record at admission so unscheduled lookups are not counted. +'''), + (' def _update_after_schedule(self, scheduler_output: SchedulerOutput) -> None:\n', + ' def _update_after_schedule(self, scheduler_output: SchedulerOutput) -> None:\n self._sparkcache_capture_prompt_steps(scheduler_output)\n'), + (' request.num_preemptions += 1\n', + ' request.num_preemptions += 1\n self._sparkcache_record_event(request, "preempted")\n'), + (' req_index = model_runner_output.req_id_to_index[req_id]\n', + ''' self._sparkcache_complete_prompt_step( + scheduler_output, request, output_is_stale + ) + req_index = model_runner_output.req_id_to_index[req_id] +'''), + (' if request.request_id in self.failed_recving_kv_req_ids:\n', + ''' cache_trace_restore_failed = request.request_id in self.failed_recving_kv_req_ids + if request.request_id in self.failed_recving_kv_req_ids: +'''), + (' self.finished_recving_kv_req_ids.remove(request.request_id)\n', + ''' self._sparkcache_record_event( + request, "restore_finalized", success=not cache_trace_restore_failed, + valid_prefix_tokens=min(request.num_computed_tokens, request.num_prompt_tokens), + ) + self.finished_recving_kv_req_ids.remove(request.request_id) +'''), + (' self._inflight_prefills.discard(request)\n connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request)\n', + ''' self._sparkcache_record_event(request, "finished", status=request.status.name) + self._inflight_prefills.discard(request) + connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request) +'''), +) + + +def transform(source: bytes) -> bytes: + if hashlib.sha256(source).hexdigest() not in SOURCE_TRANSFORMS: + raise ValueError("Cache-attribution scheduler preimage differs") + text = source.decode().replace("\r\n", "\n") + for before, after in TRANSFORMS: + if text.count(before) != 1: + raise ValueError("Cache-attribution scheduler anchor differs") + text = text.replace(before, after, 1) + ast.parse(text) + return text.encode() + + +def apply(path: Path) -> dict: + source = path.read_bytes() + before = hashlib.sha256(source).hexdigest() + for original, patched in SOURCE_TRANSFORMS.items(): + if before == patched: + return {"before_sha256": original, "after_sha256": patched} + patched = transform(source) + after = SOURCE_TRANSFORMS[before] + if hashlib.sha256(patched).hexdigest() != after: + raise ValueError("Cache-attribution scheduler postimage differs") + path.write_bytes(patched) + return {"before_sha256": before, "after_sha256": after} diff --git a/runtime/glm53-spark-mtp3-mesh/performance/attribution/test_connector_integration.py b/runtime/glm53-spark-mtp3-mesh/performance/attribution/test_connector_integration.py new file mode 100644 index 00000000..b56642c4 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/attribution/test_connector_integration.py @@ -0,0 +1,187 @@ +"""Execute scheduler observations against a companion SparkCache token ledger. + +Set SPARKCACHE_SOURCE to a SparkCache checkout containing request_attribution.py. +The companion check is optional; an explicitly selected invalid checkout fails. +""" + +import ast +import importlib.util +import os +from pathlib import Path +import sys +from types import SimpleNamespace + +import pytest + +from test_patch_scheduler import execute_statements, process_output, request, runtime +from test_patch_scheduler import scheduler_source # noqa: F401 + + +@pytest.fixture +def ledger_type(): + checkout = os.environ.get("SPARKCACHE_SOURCE") + if not checkout: + pytest.skip("SPARKCACHE_SOURCE selects the companion token ledger") + source = Path(checkout) / "sparkcache/request_attribution.py" + if not source.is_file(): + pytest.fail(f"Companion token ledger is missing: {source}") + name = "sparkcache_companion_request_attribution" + spec = importlib.util.spec_from_file_location(name, source) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module.RequestAttribution + + +def coupled(ledger_type, **overrides): + req = request(**overrides) + ledger = ledger_type(req.num_prompt_tokens) + cls, methods = runtime() + obj = cls() + obj.requests = {req.request_id: req} + summaries = [] + + def record(req, event, **fields): + if event == "finished": + summaries.append(ledger.summary(fields["status"])) + else: + ledger.record(event, **fields) + + obj.connector = SimpleNamespace(request_cache_events_enabled=True, + record_request_cache_event=record) + return obj, req, ledger, methods, summaries + + +def admit(obj, req, methods, local, external=0): + statement = next(node for node in ast.walk(methods["schedule"]) + if isinstance(node, ast.If) + and ast.unparse(node.test) == "did_prefix_cache_lookup or cache_trace_lease_attached") + execute_statements([statement], dict(self=obj, request=req, + did_prefix_cache_lookup=True, cache_trace_lease_attached=False, + num_new_local_computed_tokens=local, num_external_computed_tokens=external)) + + +def restore(obj, req, prefix, failed=False): + req.num_computed_tokens = prefix + obj.failed_recving_kv_req_ids = {req.request_id} if failed else set() + obj.finished_recving_kv_req_ids = {req.request_id} + obj.kv_cache_manager = SimpleNamespace(cache_blocks=lambda *args: None, + free=lambda *args: None) + obj.needs_kv_cache_zeroing = False + obj._update_waiting_for_remote_kv(req) + + +def dispatch(obj, req, start, count): + req.num_computed_tokens = start + output = SimpleNamespace(num_scheduled_tokens={req.request_id: count}) + obj._sparkcache_capture_prompt_steps(output) + return output + + +def finish(obj, req, summaries, status="FINISHED_STOPPED"): + req.status = SimpleNamespace(name=status) + req.is_finished = lambda: True + obj._inflight_prefills = SimpleNamespace(discard=lambda req: None) + obj._connector_finished = lambda req: (False, None) + obj.ec_connector = None + obj.encoder_cache_manager = SimpleNamespace(free=lambda req: None) + obj.finished_req_ids = set() + obj.finished_req_ids_dict = None + obj._free_blocks = lambda req: None + obj._free_request(req) + return summaries[-1] + + +def test_verified_external_prefix_and_async_output_account_once(ledger_type): + obj, req, ledger, methods, summaries = coupled(ledger_type) + admit(obj, req, methods, 256, 744) + restore(obj, req, 1000) + assert req.num_computed_tokens == 999 + output = dispatch(obj, req, 999, 32) + req.num_computed_tokens = 1100 + process_output(obj, methods, output, req) + process_output(obj, methods, output, req) + result = finish(obj, req, summaries) + assert result["attribution_complete"] + assert (result["local_tokens_reused"], result["external_tokens_reused"], + result["prompt_tokens_computed"]) == (256, 743, 1) + + +def test_failed_restore_and_same_generation_local_readmission(ledger_type): + obj, req, ledger, methods, summaries = coupled(ledger_type) + admit(obj, req, methods, 0, 1000) + restore(obj, req, 0, failed=True) + admit(obj, req, methods, 512) + output = dispatch(obj, req, 512, 488) + process_output(obj, methods, output, req) + result = finish(obj, req, summaries) + assert result["attribution_complete"] + assert (result["local_tokens_reused"], result["external_tokens_reused"], + result["prompt_tokens_computed"]) == (512, 0, 488) + + +def test_preemption_preserves_accepted_work_and_discards_inflight_output(ledger_type): + obj, req, ledger, methods, summaries = coupled( + ledger_type, status="running", spec_token_ids=[], num_output_placeholders=0) + admit(obj, req, methods, 256) + output = dispatch(obj, req, 256, 256) + process_output(obj, methods, output, req) + stale_output = dispatch(obj, req, 512, 32) + req.num_in_flight_tokens = 32 + obj._free_request_blocks = lambda req: None + obj.encoder_cache_manager = SimpleNamespace(free=lambda req: None) + obj._inflight_prefills = SimpleNamespace(discard=lambda req: None) + obj.log_stats = False + obj.waiting = SimpleNamespace(prepend_request=lambda req: None) + obj.reset_preempted_req_ids = set() + obj._preempt_request(req, 1.0) + process_output(obj, methods, stale_output, req) + admit(obj, req, methods, 0) + output = dispatch(obj, req, 0, 1000) + process_output(obj, methods, output, req) + result = finish(obj, req, summaries) + assert result["attribution_complete"] + assert result["preemptions"] == 1 + assert (result["local_tokens_reused"], result["external_tokens_reused"], + result["prompt_tokens_computed"]) == (256, 0, 1256) + + +def test_abort_does_not_credit_a_materialized_external_prefix(ledger_type): + obj, req, ledger, methods, summaries = coupled(ledger_type) + admit(obj, req, methods, 0, 1000) + restore(obj, req, 1000) + output = dispatch(obj, req, 999, 32) + req.is_finished = lambda: True + process_output(obj, methods, output, req) + result = finish(obj, req, summaries, "FINISHED_ABORTED") + assert not result["attribution_complete"] + assert result["external_tokens_reused"] == result["prompt_tokens_computed"] == 0 + + +def test_deferred_gpu_lease_admission_is_credited_only_after_execution(ledger_type): + obj, req, ledger, methods, summaries = coupled( + ledger_type, num_computed_tokens=0, prefill_stats=None, has_encoder_inputs=False) + nodes = list(ast.walk(methods["schedule"])) + initialize = next(node for node in nodes if isinstance(node, ast.Assign) + and ast.unparse(node.targets[0]) == "cache_trace_lease_attached") + attach = next(node for node in nodes if isinstance(node, ast.If) + and ast.unparse(node.test) == "attached_tokens") + allocate = next(node for node in nodes if isinstance(node, ast.If) + and ast.unparse(node.test) == "new_blocks is None") + admission = next(node for node in nodes if isinstance(node, ast.If) + and ast.unparse(node.test) == "did_prefix_cache_lookup or cache_trace_lease_attached") + values = dict(self=obj, request=req, request_id=req.request_id, lease_key="lease", + attached_tokens=768, did_prefix_cache_lookup=False, + num_new_local_computed_tokens=0, num_external_computed_tokens=0, + new_blocks=None) + execute_statements([initialize, attach, allocate, admission], values) + assert ledger.attempt is None + values["new_blocks"] = object() + execute_statements([initialize, allocate, admission], values) + assert ledger.local_tokens_reused == ledger.external_tokens_reused == 0 + output = dispatch(obj, req, req.num_computed_tokens, 232) + process_output(obj, methods, output, req) + result = finish(obj, req, summaries) + assert result["attribution_complete"] + assert (result["local_tokens_reused"], result["external_tokens_reused"], + result["prompt_tokens_computed"]) == (768, 0, 232) diff --git a/runtime/glm53-spark-mtp3-mesh/performance/attribution/test_patch_scheduler.py b/runtime/glm53-spark-mtp3-mesh/performance/attribution/test_patch_scheduler.py new file mode 100644 index 00000000..821b6476 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/attribution/test_patch_scheduler.py @@ -0,0 +1,253 @@ +"""Execute attribution seams extracted from the transformed scheduler source.""" + +import ast +import hashlib +import importlib.util +from pathlib import Path +from types import SimpleNamespace +import tarfile + +import pytest + +HERE = Path(__file__).resolve().parent +SPEC = importlib.util.spec_from_file_location("cache_attribution_patch", HERE / "patch_scheduler.py") +PATCH = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(PATCH) +SOURCE = HERE.parent / "checkpoints/payload-by-sha" / PATCH.BEFORE_SHA256 / "scheduler.py" + + +@pytest.fixture(autouse=True, params=["fresh_prompt", "continuation"]) +def scheduler_source(request, monkeypatch, tmp_path): + if request.param == "continuation": + with tarfile.open(HERE.parent / "continuation/source.tar.gz") as archive: + data = archive.extractfile("vllm/v1/core/sched/scheduler.py").read() + source = tmp_path / "continuation-scheduler.py" + source.write_bytes(data) + monkeypatch.setitem(globals(), "SOURCE", source) + + +def runtime(): + tree = ast.parse(PATCH.transform(SOURCE.read_bytes())) + cls = next(node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "Scheduler") + methods = {node.name: node for node in cls.body if isinstance(node, ast.FunctionDef)} + chosen = [node for name, node in methods.items() if name.startswith("_sparkcache_")] + chosen.extend(methods[name] for name in ("_update_waiting_for_remote_kv", "_preempt_request", "_free_request")) + module = ast.Module(body=[ast.ImportFrom(module="__future__", names=[ast.alias(name="annotations")], level=0), + ast.ClassDef(name="Harness", bases=[], keywords=[], body=chosen, decorator_list=[])], type_ignores=[]) + namespace = {"RequestStatus": SimpleNamespace(RUNNING="running", PREEMPTED="preempted")} + exec(compile(ast.fix_missing_locations(module), "scheduler-seams", "exec"), namespace) + return namespace["Harness"], methods + + +def execute_statements(statements, values): + # The scheduler's continue statements retain their real control flow. + body = [ast.For(target=ast.Name(id="_once", ctx=ast.Store()), + iter=ast.Tuple(elts=[ast.Constant(1)], ctx=ast.Load()), + body=statements, orelse=[])] + module = ast.Module(body=body, type_ignores=[]) + exec(compile(ast.fix_missing_locations(module), "scheduler-statements", "exec"), values) + + +def request(**overrides): + values = dict(request_id="request", num_computed_tokens=768, num_prompt_tokens=1000, + num_tokens=1000, num_preemptions=0, num_in_flight_tokens=32, + num_stale_output_tokens=0, drop_stale_output=False, + is_finished=lambda: False) + values.update(overrides) + return SimpleNamespace(**values) + + +def scheduler(req): + cls, methods = runtime() + events = [] + obj = cls() + obj.requests = {req.request_id: req} + obj.connector = SimpleNamespace(request_cache_events_enabled=True, + record_request_cache_event=lambda req, event, **fields: events.append((event, fields))) + return obj, events, methods + + +def test_transform_exact_source_and_idempotence(tmp_path): + target = tmp_path / "scheduler.py" + target.write_bytes(SOURCE.read_bytes()) + expected = PATCH.SOURCE_TRANSFORMS[hashlib.sha256(SOURCE.read_bytes()).hexdigest()] + PATCH.apply(target) + assert hashlib.sha256(target.read_bytes()).hexdigest() == expected + assert PATCH.apply(target)["after_sha256"] == expected + target.write_bytes(target.read_bytes() + b"# unsupported\n") + with pytest.raises(ValueError, match="preimage"): + PATCH.apply(target) + + +@pytest.mark.parametrize("local,external,lease,lookup,expected", [ + (767, 0, False, True, 767), # Retained partial tail, not rounded connector argument. + (512, 488, False, True, 512), + (1024, 512, False, True, 1000), + (512, 0, True, False, 768), + (0, 0, False, False, None), # Resume after restore is not another admission. +]) +def test_admission_uses_adopted_prefix_not_lookup_offer(local, external, lease, lookup, expected): + req = request() + obj, events, methods = scheduler(req) + statement = next(node for node in ast.walk(methods["schedule"]) + if isinstance(node, ast.If) and ast.unparse(node.test) == "did_prefix_cache_lookup or cache_trace_lease_attached") + execute_statements([statement], dict(self=obj, request=req, + did_prefix_cache_lookup=lookup, cache_trace_lease_attached=lease, + num_new_local_computed_tokens=local, num_external_computed_tokens=external)) + if expected is None: + assert events == [] + else: + assert events == [("admitted", dict(local_tokens=expected, external_tokens=min(external,1000-expected), + lease_attached=lease, source="gpu_lease" if lease else "prefix_lookup", preemptions=0))] + + +def process_output(obj, methods, output, req, failed=False): + loop = next(node for node in ast.walk(methods["update_from_output"]) + if isinstance(node, ast.For) and ast.unparse(node.target) == "(req_id, num_tokens_scheduled)") + # Execute the real failure, abort and stale-output checks through the hook. + end = next(i for i, node in enumerate(loop.body) + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Attribute) + and node.value.func.attr == "_sparkcache_complete_prompt_step") + execute_statements(loop.body[:end + 1], dict(self=obj, scheduler_output=output, + req_id=req.request_id, num_tokens_scheduled=32, + failed_kv_load_req_ids={req.request_id} if failed else set())) + + +def test_completed_ranges_survive_async_counter_advance_and_exclude_decode(): + req = request(num_computed_tokens=992) + obj, events, methods = scheduler(req) + output = SimpleNamespace(num_scheduled_tokens={req.request_id: 32}) + obj._sparkcache_capture_prompt_steps(output) + req.num_computed_tokens = 1100 + process_output(obj, methods, output, req) + assert events == [("prompt_step_completed", dict(start_token=992, end_token=1000, + preemptions=0, stale=False))] + obj._sparkcache_complete_prompt_step(output, req, False) + assert len(events) == 1 + + +@pytest.mark.parametrize("failure", ["invalid_restore", "abort", "stale", "stale_drop", "preempted"]) +def test_failed_aborted_and_stale_output_cannot_credit_prompt_work(failure): + req = request() + obj, events, methods = scheduler(req) + output = SimpleNamespace(num_scheduled_tokens={req.request_id: 32}) + obj._sparkcache_capture_prompt_steps(output) + if failure == "abort": + req.is_finished = lambda: True + if failure in ("stale", "stale_drop"): + req.num_stale_output_tokens = 32 + req.drop_stale_output = failure == "stale_drop" + if failure == "preempted": + req.num_preemptions = 1 + process_output(obj, methods, output, req, failed=failure == "invalid_restore") + assert events == [] + + +def test_first_decode_output_can_commit_admitted_reuse_without_prompt_compute(): + req = request(num_computed_tokens=1024, num_preemptions=1) + obj, events, methods = scheduler(req) + output = SimpleNamespace(num_scheduled_tokens={req.request_id: 32}) + obj._sparkcache_capture_prompt_steps(output) + process_output(obj, methods, output, req) + assert events[0][1] == dict(start_token=1000, end_token=1000, preemptions=1, stale=False) + + +@pytest.mark.parametrize("failed,valid,expected", [(False, 1000, 999), (True, 0, 0), (True, 512, 512)]) +def test_restore_finalization_reports_effective_prefix_after_clamp_or_failure(failed, valid, expected): + req = request(num_computed_tokens=valid) + obj, events, _ = scheduler(req) + obj.failed_recving_kv_req_ids = {req.request_id} if failed else set() + obj.finished_recving_kv_req_ids = {req.request_id} + obj.kv_cache_manager = SimpleNamespace(cache_blocks=lambda *args: None, free=lambda *args: None) + obj.needs_kv_cache_zeroing = False + obj._update_waiting_for_remote_kv(req) + assert events == [("restore_finalized", dict(success=not failed, + valid_prefix_tokens=expected, preemptions=0))] + assert not obj.finished_recving_kv_req_ids + + +def test_unknown_or_disabled_connector_needs_no_accounting_metadata(): + req = request() + obj, events, _ = scheduler(req) + obj.connector = SimpleNamespace() + output = SimpleNamespace(num_scheduled_tokens={req.request_id: 32}) + obj._sparkcache_capture_prompt_steps(output) + obj._sparkcache_record_event(req, "admitted") + assert not hasattr(output, "_sparkcache_prompt_steps") + assert events == [] + + +def test_preemption_and_terminal_hooks_keep_attempt_generation_and_status(): + req = request(status="running", spec_token_ids=[], num_output_placeholders=0) + obj, events, _ = scheduler(req) + obj._free_request_blocks = lambda req: None + obj.encoder_cache_manager = SimpleNamespace(free=lambda req: None) + obj._inflight_prefills = SimpleNamespace(discard=lambda req: None) + obj.log_stats = False + obj.waiting = SimpleNamespace(prepend_request=lambda req: None) + obj.reset_preempted_req_ids = set() + obj._preempt_request(req, 1.0) + assert req.num_computed_tokens == 0 + assert events == [("preempted", {"preemptions": 1})] + req.status = SimpleNamespace(name="FINISHED_ABORTED") + req.is_finished = lambda: True + obj._connector_finished = lambda req: (False, None) + obj.ec_connector = None + obj.finished_req_ids = set() + obj.finished_req_ids_dict = None + obj._free_blocks = lambda req: None + obj._free_request(req) + assert events[-1] == ("finished", {"status": "FINISHED_ABORTED", "preemptions": 1}) + + +def test_lease_attribution_survives_allocation_deferral_until_accepted_output(): + req = request(num_computed_tokens=0, prefill_stats=None, has_encoder_inputs=False) + obj, events, methods = scheduler(req) + schedule_nodes = list(ast.walk(methods["schedule"])) + initialize = next(node for node in schedule_nodes if isinstance(node, ast.Assign) + and ast.unparse(node.targets[0]) == "cache_trace_lease_attached") + attach = next(node for node in schedule_nodes if isinstance(node, ast.If) + and ast.unparse(node.test) == "attached_tokens") + allocate = next(node for node in schedule_nodes if isinstance(node, ast.If) + and ast.unparse(node.test) == "new_blocks is None") + admit = next(node for node in schedule_nodes if isinstance(node, ast.If) + and ast.unparse(node.test) == "did_prefix_cache_lookup or cache_trace_lease_attached") + values = dict(self=obj, request=req, request_id=req.request_id, lease_key="lease", + attached_tokens=768, did_prefix_cache_lookup=False, + num_new_local_computed_tokens=0, num_external_computed_tokens=0, + new_blocks=None) + execute_statements([initialize, attach, allocate, admit], values) + assert req.num_computed_tokens == 768 + assert req._sparkcache_pending_lease_generation == 0 + assert events == [] + values["new_blocks"] = object() + execute_statements([initialize, allocate, admit], values) + assert events[0] == ("admitted", dict(local_tokens=768, external_tokens=0, + lease_attached=True, source="gpu_lease", preemptions=0)) + assert req._sparkcache_pending_lease_generation is None + execute_statements([initialize, allocate, admit], values) + assert len(events) == 1 + output = SimpleNamespace(num_scheduled_tokens={req.request_id: 32}) + obj._sparkcache_capture_prompt_steps(output) + process_output(obj, methods, output, req) + assert events[-1] == ("prompt_step_completed", dict(start_token=768, end_token=800, + preemptions=0, stale=False)) + + +def test_ordinary_decode_avoids_repeated_empty_prompt_events(): + req = request(num_computed_tokens=1000) + obj, events, methods = scheduler(req) + first = SimpleNamespace(num_scheduled_tokens={req.request_id: 32}) + queued = SimpleNamespace(num_scheduled_tokens={req.request_id: 32}) + obj._sparkcache_capture_prompt_steps(first) + obj._sparkcache_capture_prompt_steps(queued) + process_output(obj, methods, first, req) + obj._sparkcache_complete_prompt_step(queued, req, False) + assert len(events) == 1 + following = SimpleNamespace(num_scheduled_tokens={req.request_id: 32}) + obj._sparkcache_capture_prompt_steps(following) + assert following._sparkcache_prompt_steps == {} + req.num_preemptions = 1 + obj._sparkcache_capture_prompt_steps(following) + assert following._sparkcache_prompt_steps[req.request_id] == (1000, 1000, 1) diff --git a/runtime/glm53-spark-mtp3-mesh/performance/continuation/LICENSE b/runtime/glm53-spark-mtp3-mesh/performance/continuation/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/continuation/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/runtime/glm53-spark-mtp3-mesh/performance/continuation/NOTICE b/runtime/glm53-spark-mtp3-mesh/performance/continuation/NOTICE new file mode 100644 index 00000000..5e7e22b1 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/continuation/NOTICE @@ -0,0 +1,4 @@ +The archived source files derive from vLLM and preserve their Apache-2.0 +SPDX notices and vLLM contributor copyright statements. SparkRing changes +implement final-chunk recurrent checkpoint planning and allocation. +The manifest identifies exact source preimages and replacements. diff --git a/runtime/glm53-spark-mtp3-mesh/performance/continuation/README.md b/runtime/glm53-spark-mtp3-mesh/performance/continuation/README.md new file mode 100644 index 00000000..f5c48c05 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/continuation/README.md @@ -0,0 +1,72 @@ +# Final-chunk recurrent checkpoints + +Status: **research-only**. The source package preserves the four continuation +files used by the serving image identified below. CPU tests verify package +integrity and ownership updates; they do not qualify the combined rebuilt image. + +[Bounded serving evidence](../../../../performance/records/glm53-flash/continuation-checkpoints-20260906/README.md) +records cold 8K–128K timing, GPU state checks, cache-reuse limits, and the exact +image configuration. Those observations have no paired-control speedup claim. + +`source.tar.gz` contains exact vLLM source bytes for: + +- `vllm/v1/core/kv_cache_manager.py` +- `vllm/v1/core/recurrent_prefill_checkpoint.py` +- `vllm/v1/core/sched/scheduler.py` +- `vllm/v1/core/single_type_kv_cache_manager.py` + +The [manifest](manifest.json) binds the archive hash, each source preimage and +replacement, and the originating overlay manifest. The archived files preserve +their upstream notices; [LICENSE](LICENSE) contains the Apache 2.0 license and +[NOTICE](NOTICE) identifies the local modifications. Compression prevents text +checkout conversion and formatters from changing source bytes. + +## Behavior + +With `SPARK_GDN_PREFILL_CHECKPOINTS=2` and +`SPARK_GDN_CONTINUATION_CHECKPOINTS=1`, the scheduler can retain explicit +recurrent checkpoints inside the final continuation chunk of a cold text +prefill. The chunk is capped at 8,192 tokens. Eligibility starts only after +successful uncached admission and is removed on cleanup or invalidation. +Preempted, resumed, encoder, speculative, and incompatible request states do +not use this path. + +The allocator validates the recurrent source and private speculative reserve, +preserves worker-visible block IDs, and allocates checkpoint destinations +without duplicate ownership. Publication boundaries must lie strictly inside +the selected chunk; an existing boundary at the chunk start is not a new export. + +## Provenance and composition + +The four files match the continuation serving image with config ID +`sha256:489d1975619e9083d14f14bcd1c6cbb4a96c41e3ab3978af048dc3ed2bb452a8`. +Its parent was the published cache/checkpoint image with config ID +`sha256:6921a6c163ea40b603e19a0332330efe3dbccbf4dce9f6cbbf6b756c9231835a`. +These identities establish source provenance, not qualification of another +image containing additional changes. + +The performance image installer validates all checkpoint ownership dependencies +before applying these four source replacements. `install.py` verifies the +manifest, archive, source preimages, and corresponding ownership entries before +writing any source file. It updates only those four ownership hashes and +preserves every symbol requirement and unrelated dependency. + +[Request-attribution instrumentation](../attribution/README.md) then applies its +separately pinned scheduler transform. The builder generates a complete installed +file inventory after both transforms. It does not copy the originating image's +performance or ownership receipts over the combined installation. Published +image contracts retain their immutable identities. + +## Offline checks + +```bash +python -m pytest runtime/glm53-spark-mtp3-mesh/performance/continuation -q +``` + +The tests reject altered packages, unexpected runtime preimages, and ownership +drift before any source write. Attribution tests execute scheduler boundaries +against both the fresh-prompt and continuation source variants. +The runtime regressions execute packaged scheduler/allocator methods to cover +source retention, speculative-reserve overlap, allocation failure, publication +pins, worker-visible block IDs, admission provenance, and cleanup. The 6K +scheduling case is CPU-only coverage, not serving qualification at that size. diff --git a/runtime/glm53-spark-mtp3-mesh/performance/continuation/install.py b/runtime/glm53-spark-mtp3-mesh/performance/continuation/install.py new file mode 100644 index 00000000..1d4f525a --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/continuation/install.py @@ -0,0 +1,57 @@ +"""Install four exact continuation sources after checkpoint ownership verification.""" + +import hashlib +import io +import json +from pathlib import Path +import tarfile + +HERE = Path(__file__).resolve().parent +MANIFEST_SHA256 = "ba458f84d1c07f5e070620629afc1259ca206feffb23ea9bf4f00931f79fbe7a" + + +def package(context=HERE): + raw = (context / "manifest.json").read_bytes() + if hashlib.sha256(raw).hexdigest() != MANIFEST_SHA256: + raise ValueError("Continuation source manifest differs") + manifest = json.loads(raw) + compressed = (context / "source.tar.gz").read_bytes() + if hashlib.sha256(compressed).hexdigest() != manifest["source_archive_sha256"]: + raise ValueError("Continuation source archive differs") + sources = {} + with tarfile.open(fileobj=io.BytesIO(compressed), mode="r:gz") as archive: + for member in archive.getmembers(): + if not member.isfile() or member.name not in manifest["files"] or member.name in sources: + raise ValueError("Continuation archive member differs") + content = archive.extractfile(member).read() + if hashlib.sha256(content).hexdigest() != manifest["files"][member.name]["after_sha256"]: + raise ValueError("Continuation source payload differs") + compile(content, member.name, "exec") + sources[member.name] = content + if sources.keys() != manifest["files"].keys(): + raise ValueError("Continuation source inventory differs") + return manifest, sources + + +def apply(site: Path, ownership: dict, context=HERE): + manifest, sources = package(context) + rows = {row["path"]: row for row in ownership["files"]} + if len(rows) != len(ownership["files"]): + raise ValueError("Duplicate ownership dependency") + writes = [] + for name, source in sources.items(): + expected = manifest["files"][name] + target = site / name + if not target.resolve().is_relative_to(site.resolve()): + raise ValueError("Continuation source target escapes installation") + if hashlib.sha256(target.read_bytes()).hexdigest() != expected["before_sha256"]: + raise ValueError(f"Continuation runtime preimage differs: {name}") + if name not in rows or rows[name]["sha256"] != expected["before_sha256"]: + raise ValueError(f"Continuation ownership preimage differs: {name}") + writes.append((target, source)) + # Validate all four source and ownership preimages before the first write. + for target, source in writes: + target.write_bytes(source) + for name, expected in manifest["files"].items(): + rows[name]["sha256"] = expected["after_sha256"] + return {"manifest_sha256": MANIFEST_SHA256, "files": manifest["files"]} diff --git a/runtime/glm53-spark-mtp3-mesh/performance/continuation/manifest.json b/runtime/glm53-spark-mtp3-mesh/performance/continuation/manifest.json new file mode 100644 index 00000000..629ef46e --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/continuation/manifest.json @@ -0,0 +1,30 @@ +{ + "environment": { + "SPARK_GDN_CONTINUATION_CHECKPOINTS": "1", + "SPARK_GDN_PREFILL_CHECKPOINTS": "2" + }, + "files": { + "vllm/v1/core/kv_cache_manager.py": { + "after_sha256": "26da2ce150958b9eb1495badfa5493f2164f8d86b943306b18b6b2abf83189c3", + "before_sha256": "f4ccf9da197eb68fd21b23e8c03615cd0d4c2f871da0dc32a9c6ec82e44d3931" + }, + "vllm/v1/core/recurrent_prefill_checkpoint.py": { + "after_sha256": "e29068047dbc505df14888f68f56745545873d3665c1db9640a53234c6a420dc", + "before_sha256": "51ef011d11bb374d61840f1ff819759b6e3dd325da41856b7b0f01b013b54713" + }, + "vllm/v1/core/sched/scheduler.py": { + "after_sha256": "f46c40c1c41daf2bab4566dd185d320f4ec1fb11e8d632c90c47b0f1bb1808fa", + "before_sha256": "ce9460834e08f97dbbfeb3f1238b78ee6a3363dd59a2aef8ceeb715857385895" + }, + "vllm/v1/core/single_type_kv_cache_manager.py": { + "after_sha256": "198dbd75397903d1e3378ac8af0ac0cd3f6b85fbb0f7bd5c65cd1780b2460a26", + "before_sha256": "10a6a60f31c67b59825aa827084f3d25416128ba5dc7c14a774675d64bbec40b" + } + }, + "parent_image_id": "sha256:6921a6c163ea40b603e19a0332330efe3dbccbf4dce9f6cbbf6b756c9231835a", + "schema": "sparkring-continuation-source/v1", + "source_archive_sha256": "91a411772951ef52f38e1d5224cd814e09ab9978d77a8fddcd665c9d5dfebf0c", + "source_overlay_manifest_sha256": "9e6b368508c30a4f443cb16da25230c7d982792aea1d99e90dd5f8f21e5c8429", + "source_role": "Final-chunk recurrent checkpoints for cold text prefills", + "status": "research-only" +} diff --git a/runtime/glm53-spark-mtp3-mesh/performance/continuation/source.tar.gz b/runtime/glm53-spark-mtp3-mesh/performance/continuation/source.tar.gz new file mode 100644 index 00000000..c3f834e3 Binary files /dev/null and b/runtime/glm53-spark-mtp3-mesh/performance/continuation/source.tar.gz differ diff --git a/runtime/glm53-spark-mtp3-mesh/performance/continuation/test_install.py b/runtime/glm53-spark-mtp3-mesh/performance/continuation/test_install.py new file mode 100644 index 00000000..d6f1f853 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/continuation/test_install.py @@ -0,0 +1,67 @@ +"""Verify exact continuation inputs and ownership updates without GPU access.""" + +import copy +import hashlib +import importlib.util +import json +from pathlib import Path +import shutil + +import pytest + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("continuation_source_install", HERE / "install.py") +INSTALL = importlib.util.module_from_spec(spec) +spec.loader.exec_module(INSTALL) + + +def fixture(tmp_path): + manifest, sources = INSTALL.package() + ownership = json.loads((HERE.parent / "checkpoints/ownership-contract.json").read_text()) + for name, row in manifest["files"].items(): + baseline = HERE.parent / "checkpoints/payload-by-sha" / row["before_sha256"] / Path(name).name + assert hashlib.sha256(baseline.read_bytes()).hexdigest() == row["before_sha256"] + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(baseline.read_bytes()) + return manifest, sources, ownership + + +def test_exact_sources_replace_only_four_verified_ownership_rows(tmp_path): + manifest, sources, ownership = fixture(tmp_path) + original = copy.deepcopy(ownership) + result = INSTALL.apply(tmp_path, ownership) + assert len(sources) == 4 + assert result["manifest_sha256"] == INSTALL.MANIFEST_SHA256 + for before, after in zip(original["files"], ownership["files"]): + if before["path"] in sources: + assert after["sha256"] == manifest["files"][before["path"]]["after_sha256"] + assert (tmp_path / before["path"]).read_bytes() == sources[before["path"]] + assert {k: v for k, v in after.items() if k != "sha256"} == {k: v for k, v in before.items() if k != "sha256"} + else: + assert after == before + + +@pytest.mark.parametrize("corrupt", ["runtime", "ownership"]) +def test_all_preimages_checked_before_any_runtime_write(tmp_path, corrupt): + manifest, sources, ownership = fixture(tmp_path) + target = list(sources)[-1] + if corrupt == "runtime": + (tmp_path / target).write_bytes(b"unsupported") + else: + next(row for row in ownership["files"] if row["path"] == target)["sha256"] = "0" * 64 + before = {name: (tmp_path / name).read_bytes() for name in sources} + original = copy.deepcopy(ownership) + with pytest.raises(ValueError, match="preimage differs"): + INSTALL.apply(tmp_path, ownership) + assert ownership == original + assert before == {name: (tmp_path / name).read_bytes() for name in sources} + + +@pytest.mark.parametrize("name", ["manifest.json", "source.tar.gz"]) +def test_modified_source_package_is_rejected(tmp_path, name): + shutil.copytree(HERE, tmp_path / "context", ignore=shutil.ignore_patterns("__pycache__")) + path = tmp_path / "context" / name + path.write_bytes(path.read_bytes() + b"unexpected") + with pytest.raises(ValueError, match="differs"): + INSTALL.package(tmp_path / "context") diff --git a/runtime/glm53-spark-mtp3-mesh/performance/continuation/test_runtime.py b/runtime/glm53-spark-mtp3-mesh/performance/continuation/test_runtime.py new file mode 100644 index 00000000..109d4fe6 --- /dev/null +++ b/runtime/glm53-spark-mtp3-mesh/performance/continuation/test_runtime.py @@ -0,0 +1,619 @@ +"""CPU tests of candidate methods, not live model or CUDA qualification.""" + +import ast +from collections import defaultdict +from dataclasses import dataclass +import importlib.util +from pathlib import Path +from types import SimpleNamespace as NS, ModuleType +import sys +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).parent +spec = importlib.util.spec_from_file_location( + "continuation_regression_installer", ROOT / "install.py" +) +installer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(installer) +_, sources = installer.package() +helper = ModuleType("continuation_regression_helper") +exec( + compile( + sources["vllm/v1/core/recurrent_prefill_checkpoint.py"], + "recurrent_prefill_checkpoint.py", + "exec", + ), + helper.__dict__, +) + + +class MambaSpec: + block_size = 512 + num_prefill_checkpoint_blocks = 2 + + +def extract(path, name, methods, scope, base=None): + cls = next( + n + for n in ast.parse(sources["vllm/" + path]).body + if isinstance(n, ast.ClassDef) and n.name == name + ) + cls.body = [ + n + for n in cls.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name in methods + ] + cls.bases = [ast.Name(id=base, ctx=ast.Load())] if base else [] + cls.decorator_list = [] + module = ast.Module( + body=[ + ast.ImportFrom( + module="__future__", names=[ast.alias(name="annotations")], level=0 + ), + cls, + ], + type_ignores=[], + ) + exec(compile(ast.fix_missing_locations(module), str(ROOT / path), "exec"), scope) + return scope[name] + + +@dataclass +class Block: + block_id: int + is_null: bool = False + ref_cnt: int = 1 + block_hash: object = None + block_hash_num_tokens: int = 0 + state: object = None + + +class Pool: + hash_block_size = 512 + + def __init__(self): + self.next = 1 + self.allocated = [] + self.freed = [] + self.fail = False + self.registered = [] + + def get_new_blocks(self, count): + if self.fail: + raise RuntimeError("injected pool failure") + blocks = [Block(i) for i in range(self.next, self.next + count)] + self.next += count + self.allocated.extend(blocks) + return blocks + + def free_blocks(self, blocks): + for block in blocks: + if block.is_null: + continue + assert block.ref_cnt > 0 + block.ref_cnt -= 1 + self.freed.append(block.block_id) + + def cache_full_blocks( + self, + *, + request, + blocks, + num_cached_blocks, + num_full_blocks, + block_size, + kv_cache_group_id, + block_mask, + ): + for column in range(num_cached_blocks, num_full_blocks): + block = blocks[column] + if block.is_null or ( + block_mask is not None and not block_mask[column - num_cached_blocks] + ): + continue + block.block_hash = (column, kv_cache_group_id) + block.block_hash_num_tokens = (column + 1) * block_size + self.registered.append( + (column, block.block_id, block.block_hash_num_tokens) + ) + + +def manager(speculative=3): + path = "v1/core/single_type_kv_cache_manager.py" + scope = { + "MambaSpec": MambaSpec, + "cdiv": lambda a, b: (a + b - 1) // b, + "continuation_layout": helper.continuation_layout, + "get_group_id": lambda key: key[1], + } + base = extract( + path, + "SingleTypeKVCacheManager", + { + "cache_blocks", + "remove_skipped_blocks", + "_remove_blocks_in_range", + "pop_blocks_for_free", + "free", + }, + scope, + ) + scope["Base"] = base + cls = extract( + path, + "MambaManager", + { + "get_num_blocks_to_allocate", + "allocate_new_blocks", + "remove_skipped_blocks", + "_needs_internal_checkpoint", + "get_num_skipped_tokens", + "cache_blocks", + "_cache_partial_tail_block", + "_queue_aligned_recurrent_boundary", + "reachable_block_mask", + "pop_blocks_for_free", + }, + scope, + base="Base", + ) + obj = cls() + obj.kv_cache_spec = MambaSpec() + obj.block_size = 512 + obj.num_speculative_blocks = speculative + obj.mamba_cache_mode = "align" + obj.req_to_blocks = defaultdict(list) + obj._planned_recurrent_checkpoints = {} + obj._planned_recurrent_publications = {} + obj._allocated_block_reqs = set() + obj._partial_hit_reqs = {} + obj._num_checkpoint_blocks = {} + obj.last_state_block_idx = {} + obj._null_block = Block(0, True, 0) + obj.block_pool = Pool() + obj.num_cached_block = {} + obj.cached_blocks_this_step = set() + obj.scheduler_block_size = 2048 + obj.use_eagle = True + obj.kv_cache_group_id = 7 + obj._pending_aligned_recurrent_boundaries = [] + obj.recurrent_publication_boundary = None + obj._producer_partial_tail_reqs = {} + obj._pending_partial_tail_offloads = [] + obj._has_partial_local_hit = lambda *_: False + obj._get_num_evictable_blocks = lambda _: 0 + return obj + + +def seed(obj, start): + worker = [] + for end in range(8192, start + 1, 8192): + obj.remove_skipped_blocks("r", max(0, end - 8192)) + suffix = obj.allocate_new_blocks("r", end, end) + worker.extend(suffix) + obj.req_to_blocks["r"][end // 512 - 1].state = end + return worker + + +def request(prompt, start=0, identifier="r"): + return NS( + request_id=identifier, + num_computed_tokens=start, + num_prompt_tokens=prompt, + num_tokens=prompt, + shared_prefix_boundary=0, + has_encoder_inputs=False, + num_preemptions=0, + spec_token_ids=[], + num_in_flight_tokens=0, + status="waiting", + resumable=False, + ) + + +def scheduler(obj, enabled=True, budget=8192): + scope = { + "fresh_prompt_plan": helper.fresh_prompt_plan, + "final_chunk_plan": helper.final_chunk_plan, + "continuation_layout": helper.continuation_layout, + "MambaSpec": MambaSpec, + "RequestStatus": NS(WAITING="waiting"), + } + cls = extract( + "v1/core/sched/scheduler.py", + "Scheduler", + { + "_record_continuation_origin", + "_recurrent_checkpoint_plan", + "_mamba_block_aligned_split", + "_free_request_blocks", + }, + scope, + ) + owner = cls() + owner._two_checkpoint_prefill_enabled = True + owner._continuation_prefill_enabled = enabled + owner._continuation_prefill_origins = {} + owner.cache_config = NS(block_size=512) + owner.max_num_scheduled_tokens = budget + owner.use_eagle = True + owner.mamba_has_prefill_checkpoint_blocks = True + owner.mamba_partial_cache_hit = True + owner.hash_block_size = 512 + owner.scheduler_config = NS(long_prefill_token_threshold=0) + owner.kv_cache_manager = NS(coordinator=NS(single_type_managers=[obj])) + owner._recurrent_publication_boundaries = lambda req: tuple( + [((req.num_prompt_tokens - 1) // 2048) * 2048] + ) + return owner + + +class ContinuationTests(unittest.TestCase): + def test_fresh_8k_path_retained_when_continuation_disabled(self): + obj = manager() + owner = scheduler(obj, enabled=False) + req = request(8192) + plan = owner._recurrent_checkpoint_plan(req, 0, 8192) + self.assertEqual(plan, (0, 8192, (6144, 7168))) + obj._planned_recurrent_checkpoints["r"] = plan + self.assertEqual(obj.get_num_blocks_to_allocate("r", 8192, [], 0, 0, 8192), 6) + table = obj.allocate_new_blocks("r", 8192, 8192) + self.assertEqual( + [i for i, b in enumerate(table) if not b.is_null], [11, 13, 15, 16, 17, 18] + ) + + def test_request_release_keeps_an_extra_publication_pin(self): + for end, targets in ((10240, (9216,)), (16384, (14336, 15360))): + obj = manager() + seed(obj, 8192) + obj._planned_recurrent_checkpoints["r"] = (8192, end, targets) + obj.allocate_new_blocks("r", end, end) + published = obj.req_to_blocks["r"][targets[0] // 512 - 1] + published.state = targets[0] + published.ref_cnt += 1 + obj.free("r") + self.assertEqual(published.ref_cnt, 1) + self.assertEqual(published.state, targets[0]) + self.assertNotIn("r", obj.req_to_blocks) + self.assertNotIn("r", obj.last_state_block_idx) + self.assertEqual(len(obj.block_pool.freed), len(set(obj.block_pool.freed))) + obj.block_pool.free_blocks([published]) + self.assertEqual(published.ref_cnt, 0) + + def test_real_scheduler_and_allocator_schedules_8k_and_6k(self): + expected = { + 8192: { + 9216: [8192, 1024], + 10240: [8192, 2048], + 16384: [8192, 8192], + 32768: [8192] * 4, + }, + 6144: { + 9216: [6144, 3072], + 10240: [6144, 4096], + 16384: [6144, 6144, 4096], + 32768: [6144] * 5 + [2048], + }, + } + for budget, cases in expected.items(): + for prompt, want in cases.items(): + with self.subTest(budget=budget, prompt=prompt): + obj = manager() + owner = scheduler(obj, budget=budget) + req = request(prompt) + chunks = [] + worker = [] + while req.num_computed_tokens < prompt: + start = req.num_computed_tokens + size = owner._mamba_block_aligned_split( + req, min(budget, prompt - start) + ) + plan = owner._recurrent_checkpoint_plan( + req, start, start + size + ) + obj.remove_skipped_blocks("r", start) + if plan: + obj._planned_recurrent_checkpoints["r"] = plan + count = obj.get_num_blocks_to_allocate( + "r", start + size, [], start, start, start + size + ) + before = len(obj.block_pool.allocated) + worker.extend( + obj.allocate_new_blocks("r", start + size, start + size) + ) + self.assertEqual(len(obj.block_pool.allocated) - before, count) + # Simulated completion stamps only states this step actually produces. + if plan: + for token in plan[2]: + obj.req_to_blocks["r"][token // 512 - 1].state = token + obj.req_to_blocks["r"][(start + size) // 512 - 1].state = ( + start + size + ) + if start == 0: + owner._record_continuation_origin(req, 0, 0, 0, False) + obj._planned_recurrent_checkpoints.clear() + chunks.append(size) + req.num_computed_tokens += size + self.assertEqual(chunks, want) + self.assertEqual(worker[prompt // 512 - 1].state, prompt) + + def test_actual_allocator_source_and_worker_prefix(self): + for speculative in (0, 1, 3): + for start in (8192, 16384, 24576, 57344): + for tail in (1536, 2048, 4096, 6144, 8192): + with self.subTest(speculative=speculative, start=start, tail=tail): + obj = manager(speculative) + worker = seed(obj, start) + old = tuple(obj.req_to_blocks["r"]) + worker_before = tuple(worker) + plan = helper.final_chunk_plan( + start=start, + end=start + tail, + prompt=start + tail, + num_tokens=start + tail, + block_size=512, + publications=(((start + tail - 1) // 2048) * 2048,), + ) + obj._planned_recurrent_checkpoints["r"] = plan + admission = obj.get_num_blocks_to_allocate( + "r", start + tail, [], start, start, start + tail + ) + self.assertEqual(admission, len(plan[2]) + 1) + before = len(obj.block_pool.allocated) + suffix = obj.allocate_new_blocks( + "r", start + tail, start + tail + ) + worker.extend(suffix) + self.assertEqual( + len(obj.block_pool.allocated) - before, admission + ) + self.assertEqual( + worker[: len(worker_before)], list(worker_before) + ) + table = obj.req_to_blocks["r"] + source = start // 512 - 1 + self.assertIs(table[source], old[source]) + self.assertEqual(table[source].state, start) + active = ( + [source] + + [p // 512 - 1 for p in plan[2]] + + list(range((start + tail) // 512 - 1, len(table))) + ) + self.assertEqual( + len(active), len({worker[c].block_id for c in active}) + ) + self.assertTrue(all(worker[c] is table[c] for c in active)) + owned = [b.block_id for b in table if not b.is_null] + self.assertEqual(len(owned), len(set(owned))) + self.assertEqual(obj.last_state_block_idx["r"], source) + # An in-flight next chunk must not free its source yet. + obj.remove_skipped_blocks("r", start) + self.assertIs(obj.req_to_blocks["r"][source], old[source]) + # After that chunk is processed, ordinary cleanup may retire it. + obj.remove_skipped_blocks("r", start + tail) + self.assertTrue(obj.req_to_blocks["r"][source].is_null) + + def test_10k_reuses_existing_checkpoint_column(self): + obj = manager() + worker = seed(obj, 8192) + checkpoint = worker[17] + plan = (8192, 10240, (9216,)) + obj._planned_recurrent_checkpoints["r"] = plan + worker.extend(obj.allocate_new_blocks("r", 10240, 10240)) + self.assertIs(obj.req_to_blocks["r"][17], checkpoint) + self.assertIs(worker[17], checkpoint) + self.assertEqual(len(obj.block_pool.allocated), 6) # initial4 +new2 + + def test_failed_pool_allocation_does_not_mutate_table(self): + obj = manager() + seed(obj, 8192) + before = tuple(obj.req_to_blocks["r"]) + obj.block_pool.fail = True + obj._planned_recurrent_checkpoints["r"] = (8192, 16384, (14336, 15360)) + with self.assertRaises(RuntimeError): + obj.allocate_new_blocks("r", 16384, 16384) + self.assertEqual(tuple(obj.req_to_blocks["r"]), before) + self.assertNotIn("r", obj.last_state_block_idx) + + def test_private_reserve_and_source_guards(self): + for mutate in ( + lambda o: setattr(o.req_to_blocks["r"][16], "ref_cnt", 2), + lambda o: setattr(o.req_to_blocks["r"][17], "block_hash", ("x", 7)), + lambda o: o.req_to_blocks["r"].__setitem__(15, o._null_block), + lambda o: o.req_to_blocks["r"].__setitem__(18, o.req_to_blocks["r"][17]), + lambda o: o.req_to_blocks["r"].append(o._null_block), + ): + obj = manager() + seed(obj, 8192) + mutate(obj) + before = tuple(obj.req_to_blocks["r"]) + with self.assertRaises(ValueError): + helper.continuation_layout( + before, (8192, 16384, (14336, 15360)), 512, 3 + ) + self.assertEqual(tuple(obj.req_to_blocks["r"]), before) + + def test_scheduler_opt_in_provenance_and_fallback(self): + obj = manager() + owner = scheduler(obj) + req = request(16384) + self.assertIsNone(owner._recurrent_checkpoint_plan(req, 8192, 16384)) + owner._record_continuation_origin(req, 0, 0, 0, False) + seed(obj, 8192) + req.num_computed_tokens = 8192 + self.assertEqual(owner._mamba_block_aligned_split(req, 8192), 8192) + owner._continuation_prefill_enabled = False + self.assertEqual(owner._mamba_block_aligned_split(req, 8192), 6144) + owner._continuation_prefill_enabled = True + for field, value in [ + ("num_preemptions", 1), + ("has_encoder_inputs", True), + ("spec_token_ids", [3]), + ("num_tokens", 16385), + ("resumable", True), + ]: + old = getattr(req, field) + setattr(req, field, value) + self.assertIsNone(owner._recurrent_checkpoint_plan(req, 8192, 16384)) + setattr(req, field, old) + replacement = request(16384, 8192) + self.assertIsNone(owner._recurrent_checkpoint_plan(replacement, 8192, 16384)) + + def test_origin_rejected_for_cached_or_async_admission_and_cleared_on_free(self): + obj = manager() + owner = scheduler(obj) + req = request(16384) + for args in [(6144, 0, 6144, False), (6144, 6144, 0, False), (0, 0, 0, True)]: + owner._record_continuation_origin(req, *args) + self.assertNotIn("r", owner._continuation_prefill_origins) + owner._record_continuation_origin(req, 0, 0, 0, False) + owner.defer_block_free = False + owner.kv_cache_manager.free = lambda _: None + owner._free_request_blocks(req) + self.assertNotIn("r", owner._continuation_prefill_origins) + + def test_6k_budget_and_hard_cap(self): + self.assertIsNone( + helper.final_chunk_plan( + start=8192, + end=16384, + prompt=16384, + num_tokens=16384, + block_size=512, + publications=(14336,), + max_chunk_tokens=6144, + ) + ) + self.assertEqual( + helper.final_chunk_plan( + start=12288, + end=16384, + prompt=16384, + num_tokens=16384, + block_size=512, + publications=(14336,), + max_chunk_tokens=6144, + ), + (12288, 16384, (14336, 15360)), + ) + self.assertIsNone( + helper.final_chunk_plan( + start=8192, + end=24576, + prompt=24576, + num_tokens=24576, + block_size=512, + publications=(22528,), + max_chunk_tokens=32768, + ) + ) + + def test_cache_registration_and_interior_publication(self): + obj = manager() + seed(obj, 8192) + obj.num_cached_block["r"] = 16 + plan = (8192, 16384, (14336, 15360)) + obj._planned_recurrent_checkpoints["r"] = plan + obj._planned_recurrent_publications["r"] = (14336,) + obj.allocate_new_blocks("r", 16384, 16384) + req = request(16384, 8192) + obj.cache_blocks(req, 16384) + by_position = { + position: identifier + for _, identifier, position in obj.block_pool.registered + } + self.assertEqual(by_position[14336], obj.req_to_blocks["r"][27].block_id) + self.assertEqual(by_position[15360], obj.req_to_blocks["r"][29].block_id) + self.assertEqual( + [row[3] for row in obj._pending_aligned_recurrent_boundaries], [14336] + ) + + def test_wrapper_clears_plan_on_failed_admission(self): + obj = manager() + seed(obj, 8192) + req = request(16384, 8192) + cls = extract( + "v1/core/kv_cache_manager.py", "KVCacheManager", {"allocate_slots"}, {} + ) + owner = cls() + owner.coordinator = NS(single_type_managers=[obj]) + observed = [] + + def inner(**kwargs): + observed.append(obj._planned_recurrent_checkpoints["r"]) + return None + + owner._allocate_slots_without_checkpoint_plan = inner + with patch.dict( + sys.modules, {"vllm.v1.core.recurrent_prefill_checkpoint": helper} + ): + self.assertIsNone( + owner.allocate_slots( + req, + 8192, + recurrent_prefill_checkpoint_plan=(8192, 16384, (14336, 15360)), + recurrent_checkpoint_publications=(14336,), + ) + ) + self.assertEqual(len(observed), 1) + self.assertFalse(obj._planned_recurrent_checkpoints) + self.assertFalse(obj._planned_recurrent_publications) + + def raises(**kwargs): + raise RuntimeError("injected inner failure") + + owner._allocate_slots_without_checkpoint_plan = raises + with patch.dict( + sys.modules, {"vllm.v1.core.recurrent_prefill_checkpoint": helper} + ): + with self.assertRaises(RuntimeError): + owner.allocate_slots( + req, + 8192, + recurrent_prefill_checkpoint_plan=(8192, 16384, (14336, 15360)), + recurrent_checkpoint_publications=(14336,), + ) + for extra in ( + {"num_external_computed_tokens": 512}, + {"delay_cache_blocks": True}, + {"recurrent_checkpoint_publications": (8192,)}, + ): + with self.assertRaises(ValueError): + owner.allocate_slots( + req, + 8192, + recurrent_prefill_checkpoint_plan=(8192, 16384, (14336, 15360)), + **extra, + ) + self.assertFalse(obj._planned_recurrent_checkpoints) + self.assertFalse(obj._planned_recurrent_publications) + + def test_hooks_are_in_actual_admission_and_cleanup_sites(self): + tree = ast.parse(sources["vllm/v1/core/sched/scheduler.py"]) + methods = {n.name: n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)} + calls = [ + n + for n in ast.walk(methods["schedule"]) + if isinstance(n, ast.Call) + and isinstance(n.func, ast.Attribute) + and n.func.attr == "_record_continuation_origin" + ] + self.assertEqual(len(calls), 1) + self.assertIn( + "_continuation_prefill_origins.pop", + ast.unparse(methods["_update_requests_with_invalid_blocks"]), + ) + for call in [ + n for n in ast.walk(methods["schedule"]) if isinstance(n, ast.Call) + ]: + for keyword in call.keywords: + if keyword.arg == "recurrent_checkpoint_publications": + self.assertIn( + "checkpoint_plan[0] < p < checkpoint_plan[1]", + ast.unparse(keyword.value), + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/runtime/glm53-spark-mtp3-mesh/performance/install.py b/runtime/glm53-spark-mtp3-mesh/performance/install.py index 48038ad1..07ff602f 100644 --- a/runtime/glm53-spark-mtp3-mesh/performance/install.py +++ b/runtime/glm53-spark-mtp3-mesh/performance/install.py @@ -70,6 +70,24 @@ path = SITE / row["path"] if row["sha256"] != hashlib.sha256(path.read_bytes()).hexdigest(): raise ValueError(f"Ownership dependency differs: {row['path']}") +# Apply named transforms only after every checkpoint ownership preimage passed. +import importlib.util + +continuation_spec = importlib.util.spec_from_file_location( + "continuation_install", SOURCE / "continuation/install.py" +) +continuation = importlib.util.module_from_spec(continuation_spec) +continuation_spec.loader.exec_module(continuation) +continuation_transform = continuation.apply(SITE, data) +sys.path.insert(0, str(SOURCE / "attribution")) +from patch_scheduler import apply as apply_attribution + +attribution_transform = apply_attribution(scheduler) +for row in data["files"]: + if row["path"] == "vllm/v1/core/sched/scheduler.py": + if row["sha256"] != attribution_transform["before_sha256"]: + raise ValueError("Attribution ownership scheduler preimage differs") + row["sha256"] = attribution_transform["after_sha256"] contract.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") shutil.copytree(SOURCE / "bundle", Path("/opt/spark-sircl"), dirs_exist_ok=True) shutil.copyfile( @@ -105,6 +123,10 @@ "schema": "sparkring-mtp3-performance-image/v1", "status": "research-only", "sparkcache_commit": context["sparkcache_commit"], + "runtime_transforms": { + "continuation_checkpoints": continuation_transform, + "request_cache_attribution": attribution_transform, + }, "files": files, }, sort_keys=True, diff --git a/runtime/glm53-spark-mtp3-mesh/performance/prepare.py b/runtime/glm53-spark-mtp3-mesh/performance/prepare.py index 025569cb..592eac3e 100644 --- a/runtime/glm53-spark-mtp3-mesh/performance/prepare.py +++ b/runtime/glm53-spark-mtp3-mesh/performance/prepare.py @@ -10,7 +10,7 @@ import tarfile HERE = Path(__file__).resolve().parent -CACHE_COMMIT = "48bbd2be4a7b972e56632a2d7b934bac5460f272" +CACHE_COMMIT = "b5aca7cd3d3f7e7a14636bf6e5fa1f50a9650168" BASE_IMAGE = "ghcr.io/fujitsupolycom/sparkring-glm53-sparkcache@sha256:67dc0ae453baaae6831ccec1d259b4ef8b236a8b0dc9f747d901b95c66ec1987" BASE_ID = "sha256:2e41b1e934a85ff7c21b780532db2f0a0e978df081e52f4ae2bf11f8992fb24f" PLACEMENT = "2657cdd2e54a097c9544e4c79ae62c0646db6db123ff24e4f0c384238c3a1e8d" @@ -50,7 +50,7 @@ def prepare(cache, placement, transport, output): path = output / name path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(bundle.extractfile(member).read()) - for name in ("checkpoints", "reasoning"): + for name in ("checkpoints", "reasoning", "attribution", "continuation"): shutil.copytree( HERE / name, output / name,