From 21e9bb1363cf58f43835ab06237159ccee1af64f Mon Sep 17 00:00:00 2001 From: ahmadalguydi Date: Thu, 13 Aug 2026 10:24:33 +0300 Subject: [PATCH 1/2] test: cover fp8 block-scale dequantization --- tests/run.sh | 15 +++++ tests/test_fp8_blocks.py | 131 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 tests/test_fp8_blocks.py diff --git a/tests/run.sh b/tests/run.sh index a13529a..d9c1528 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1202,6 +1202,21 @@ fi # ------------------------------------------------------------ converter ---- head_ "converter" +# The fp8 converter path has two readers and a silent shape-preserving failure +# mode, so keep its tile mapping under a small synthetic test instead of +# relying on a multi-hour model conversion. Exit 77 is an explicit skip when +# torch is unavailable, never a pass. +if ! command -v python3 >/dev/null 2>&1; then + sk "fp8 block-scale mapping" "python3 not installed" +else + out=$(python3 tests/test_fp8_blocks.py 2>&1); rc=$? + case "$rc" in + 0) ok "fp8 block scales, partial tiles, missing companions, and reader agreement" ;; + 77) sk "fp8 block-scale mapping" "torch not installed" ;; + *) no "fp8 block-scale mapping"; printf '%s\n' "$out" | grep -E "FAIL|Error|Traceback" | head -5 ;; + esac +fi + # Resume is the one converter behaviour that cannot be checked by looking at # a finished container: it is about the partial states a crash leaves. The # quantizer is stubbed out, so this needs neither torch nor source weights. diff --git a/tests/test_fp8_blocks.py b/tests/test_fp8_blocks.py new file mode 100644 index 0000000..c75cfd9 --- /dev/null +++ b/tests/test_fp8_blocks.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 SQLite Cloud, Inc. +"""Regression checks for fp8 block-scale dequantization. + +These tests use small synthetic tensors so they do not need model weights. A +missing torch installation is an explicit skip, matching tests/run.sh's rule +that unavailable prerequisites must never look like a pass. +""" + +import json +import os +import struct +import sys +import tempfile + +try: + import torch +except ImportError: + print("SKIP: torch is not installed") + raise SystemExit(77) + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools")) +from convert import ShardReader +from mxfp4 import ST, unblock_scale + + +def expected_dequant(q, scale, block): + """Reference the tile lookup with explicit row and column indices.""" + bm, bn = block + rows = torch.arange(q.shape[0]) // bm + cols = torch.arange(q.shape[1]) // bn + return q.float() * scale[rows[:, None], cols[None, :]] + + +def write_safetensors_model(root, block, include_scale=True): + """Write the smallest safetensors model both readers can consume.""" + q = torch.tensor([[1.0, -2.0, 3.0, -4.0, 0.5], + [-1.0, 2.0, -3.0, 4.0, -0.5]], dtype=torch.float8_e4m3fn) + scale = torch.tensor([[1.0, 2.0]], dtype=torch.float32) + tensors = {"weight": ("F8_E4M3", list(q.shape), + bytes(q.contiguous().view(torch.uint8).flatten().tolist()))} + if include_scale: + tensors["weight_scale_inv"] = ("F32", list(scale.shape), + bytes(scale.contiguous().view(torch.uint8).flatten().tolist())) + + header = {} + payload = bytearray() + for name, (dtype, shape, raw) in tensors.items(): + start = len(payload) + payload.extend(raw) + header[name] = {"dtype": dtype, "shape": shape, + "data_offsets": [start, len(payload)]} + header_bytes = json.dumps(header, separators=(",", ":")).encode() + while (8 + len(header_bytes)) % 8: + header_bytes += b" " + with open(os.path.join(root, "shard.safetensors"), "wb") as f: + f.write(struct.pack(" Date: Thu, 13 Aug 2026 10:26:40 +0200 Subject: [PATCH 2/2] Run the fp8 test, and cover the reader whose guard was load-bearing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things on top of tomatotomata's checks, which are unchanged and were right: the mutants I planted in unblock_scale — returning q untouched, transposing the block dims, dropping the [:M, :N] crop, removing the shape guard — were all caught, and expected_dequant being an independent index-array implementation rather than a second call to the code under test is what gives it that. It never ran. run.sh invoked it as bare `python3`, and torch is not a system package on the machines that test this: CLAUDE.md says it is never a repo dependency and every other torch checker goes through uv. So the whole thing reported "torch not installed" and skipped everywhere, CI included — where the Linux job is the one that installs uv, which is to say the one place it does get to run. Through run_uv now, with the guard on uv rather than python3, since without uv run_uv exits 127 and the catch-all would report FAIL where a SKIP is meant. 56 passed / 0 failed / 2 skipped becomes 57 / 0 / 2. And the missing-companion check exercised ST only. Deleting ShardReader's own `raise` in convert.py, so it returns the tensor unscaled instead, kept the suite green — a silent wrong answer in the reader #26's description called out as "a second reader and was easy to miss". Both readers now, and that mutant is killed. The same mutation against ST survives, and should: without its guard, raw() still raises KeyError naming weight_scale_inv, so the contract the test asserts — refuses, and says which tensor — holds either way. That guard buys a better message, not a different decision, and asserting its exact prose would test the wording rather than the behaviour. Also wrote down the case nothing here can catch, which is the reason the tile size is read from config rather than inferred: 300 rows against 3 scale rows admits both 128 and 100, both pass every shape check, and the wrong one applies each scale to the wrong rows. Co-Authored-By: Claude Opus 5 --- tests/run.sh | 15 ++++++++++++--- tests/test_fp8_blocks.py | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/tests/run.sh b/tests/run.sh index d6b642a..7b40609 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1342,10 +1342,19 @@ head_ "converter" # mode, so keep its tile mapping under a small synthetic test instead of # relying on a multi-hour model conversion. Exit 77 is an explicit skip when # torch is unavailable, never a pass. -if ! command -v python3 >/dev/null 2>&1; then - sk "fp8 block-scale mapping" "python3 not installed" +# +# Through uv, like every other torch checker here: torch is not a dependency +# of this repo and is not a system package on the machines that run this. As +# bare `python3` the whole check reported "torch not installed" and skipped +# everywhere, CI included — where the Linux job is the one that installs uv, +# so this is exactly where it does get to run. +if ! command -v uv >/dev/null 2>&1; then + # The guard is on uv rather than python3 for the same reason: without uv + # run_uv exits 127 and the catch-all below would call that a failure. + sk "fp8 block-scale mapping" "uv not installed" else - out=$(python3 tests/test_fp8_blocks.py 2>&1); rc=$? + out=$(run_uv run --quiet --with torch --no-project \ + python tests/test_fp8_blocks.py 2>&1); rc=$? case "$rc" in 0) ok "fp8 block scales, partial tiles, missing companions, and reader agreement" ;; 77) sk "fp8 block-scale mapping" "torch not installed" ;; diff --git a/tests/test_fp8_blocks.py b/tests/test_fp8_blocks.py index c75cfd9..dbba939 100644 --- a/tests/test_fp8_blocks.py +++ b/tests/test_fp8_blocks.py @@ -6,6 +6,16 @@ These tests use small synthetic tensors so they do not need model weights. A missing torch installation is an explicit skip, matching tests/run.sh's rule that unavailable prerequisites must never look like a pass. + +The one case nothing here can catch, stated because it is the reason the tile +size is read from the checkpoint's config rather than inferred from the two +shapes: a *compatible but wrong* block size. 300 rows against 3 scale rows +admits both 128 (the truth, with a partial last tile) and 100 (a clean split). +Both satisfy the shape check below, both produce a tensor of the right size, +and the wrong one applies every scale to the wrong rows. No assertion over +shapes can separate them, which is why `unblock_scale` takes `block` as an +argument instead of deriving it — the check that matters happened before this +file was reached. """ import json @@ -81,14 +91,25 @@ def test_partial_last_row_and_column_are_cropped_after_mapping(): def test_missing_scale_companion_is_rejected(): - with tempfile.TemporaryDirectory() as root: - write_safetensors_model(root, (2, 3), include_scale=False) - try: - ST(root).tensor("weight") - except KeyError as exc: - assert "weight_scale_inv" in str(exc) - else: - raise AssertionError("fp8 tensor without its scale companion was accepted") + """Both readers refuse, not just the one that is easy to remember. + + ST and ShardReader each carry their own copy of this guard, and covering + only ST left the convert.py one free to return the tensor unscaled: a + mutation that deleted its `raise` kept the whole suite green. That is the + silent-wrong-answer this file exists to prevent, in the reader #26's own + description called out as "a second reader and was easy to miss". + """ + for label, read in (("mxfp4.ST", lambda r: ST(r).tensor("weight")), + ("convert.ShardReader", lambda r: ShardReader(r).get("weight"))): + with tempfile.TemporaryDirectory() as root: + write_safetensors_model(root, (2, 3), include_scale=False) + try: + read(root) + except KeyError as exc: + assert "weight_scale_inv" in str(exc), f"{label}: {exc}" + else: + raise AssertionError( + f"{label} accepted an fp8 tensor with no scale companion") def test_gross_scale_shape_mismatch_is_rejected():