From 78076ce2f1f6733e54dd43413f6e0483f0c13d06 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:21 -0800 Subject: [PATCH 01/20] [executorch][test-suite] Feed the Linear tests inputs with the model's 67 input features Pull Request resolved: https://github.com/pytorch/executorch/pull/24355 Eight of the fourteen `test_linear.py` models were fed 64-feature inputs, but `Model` defaults to `in_features=67`, so eager rejected them and every backend skipped them. The dtype, no-bias, batch-size and leading-batch tests now pass 67-feature inputs. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657740 @exported-using-ghexport Differential Revision: [D128254891](https://our.internmc.facebook.com/intern/diff/D128254891/) --- backends/test/suite/operators/test_linear.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backends/test/suite/operators/test_linear.py b/backends/test/suite/operators/test_linear.py index 30ae963a1ba..0dc4ef3fedb 100644 --- a/backends/test/suite/operators/test_linear.py +++ b/backends/test/suite/operators/test_linear.py @@ -41,7 +41,7 @@ class Linear(OperatorTest): def test_linear_dtype(self, flow: TestFlow, dtype) -> None: self._test_op( Model().to(dtype), - ((torch.rand(16, 64) * 10).to(dtype),), + ((torch.rand(16, 67) * 10).to(dtype),), flow, ) @@ -49,7 +49,7 @@ def test_linear_dtype(self, flow: TestFlow, dtype) -> None: def test_linear_no_bias_dtype(self, flow: TestFlow, dtype) -> None: self._test_op( Model(bias=False).to(dtype), - ((torch.rand(16, 64) * 10).to(dtype),), + ((torch.rand(16, 67) * 10).to(dtype),), flow, ) @@ -78,7 +78,7 @@ def test_linear_feature_sizes(self, flow: TestFlow) -> None: def test_linear_no_bias(self, flow: TestFlow) -> None: self._test_op( Model(bias=False), - (torch.randn(16, 64),), + (torch.randn(16, 67),), flow, ) self._test_op( @@ -90,17 +90,17 @@ def test_linear_no_bias(self, flow: TestFlow) -> None: def test_linear_batch_sizes(self, flow: TestFlow) -> None: self._test_op( Model(), - (torch.randn(8, 64),), + (torch.randn(8, 67),), flow, ) self._test_op( Model(), - (torch.randn(32, 64),), + (torch.randn(32, 67),), flow, ) self._test_op( Model(), - (torch.randn(100, 64),), + (torch.randn(100, 67),), flow, ) @@ -114,11 +114,11 @@ def test_linear_unbatched(self, flow: TestFlow) -> None: def test_linear_leading_batch(self, flow: TestFlow) -> None: self._test_op( Model(), - (torch.randn(4, 8, 64),), + (torch.randn(4, 8, 67),), flow, ) self._test_op( Model(), - (torch.randn(2, 4, 8, 64),), + (torch.randn(2, 4, 8, 67),), flow, ) From f9edf889f8f677ed1f67809a70508bb53f0ba168 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:22 -0800 Subject: [PATCH 02/20] [executorch][native] Split engine host factories out of the PTN runners Pull Request resolved: https://github.com/pytorch/executorch/pull/24356 Drivers other than `ptn_runner_main` need to link an engine without its runner binary. Each engine's `make_engine_host()` moves into a `*_engine_host` library and `PtnRunner.h` into `engine_host`; the runners are unchanged. Authored with Claude Code. Differential Revision: [D128254879](https://our.internmc.facebook.com/intern/diff/D128254879/) **NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D128254879/)! ghstack-source-id: 463657752 --- backends/native/test/runtime/BUCK | 14 ++++++++--- backends/native/test/runtime/PtnRunner.h | 4 ++-- backends/native/test/runtime/ggml/BUCK | 23 +++++++++++++++---- .../test/runtime/ggml/ggml_ptn_runner.cpp | 2 +- .../native/test/runtime/vulkan/targets.bzl | 12 ++++++++-- .../test/runtime/vulkan/vulkan_ptn_runner.cpp | 2 +- 6 files changed, 44 insertions(+), 13 deletions(-) diff --git a/backends/native/test/runtime/BUCK b/backends/native/test/runtime/BUCK index 80136220925..8ce73cd9078 100644 --- a/backends/native/test/runtime/BUCK +++ b/backends/native/test/runtime/BUCK @@ -8,16 +8,24 @@ non_fbcode_target(_kind = define_common_targets) fbcode_target(_kind = define_common_targets) -# Engine-neutral PTN runner; each engine's runner binary adds make_engine_host(). +# Declares make_engine_host(), which each engine's host library defines. +fbcode_target( + _kind = runtime.cxx_library, + name = "engine_host", + exported_headers = ["PtnRunner.h"], + visibility = ["//executorch/backends/native/test/..."], + exported_deps = ["//executorch/backends/native/runtime/engine:engine"], +) + +# Engine-neutral PTN runner; each engine's runner binary adds its host library. fbcode_target( _kind = runtime.cxx_library, name = "ptn_runner_main", srcs = ["ptn_runner_main.cpp"], - exported_headers = ["PtnRunner.h"], link_whole = True, undefined_symbols = True, visibility = ["//executorch/backends/native/test/..."], - exported_deps = ["//executorch/backends/native/runtime/engine:engine"], + exported_deps = [":engine_host"], deps = [ "//executorch/backends/native/runtime:runtime", "//executorch/backends/native/runtime/deserialize:package", diff --git a/backends/native/test/runtime/PtnRunner.h b/backends/native/test/runtime/PtnRunner.h index 830b7fa4f62..de23aeb4da4 100644 --- a/backends/native/test/runtime/PtnRunner.h +++ b/backends/native/test/runtime/PtnRunner.h @@ -12,8 +12,8 @@ namespace ptn::testing { -// Defined once per runner binary; ptn_runner_main.cpp drives the host it -// returns, so every engine runs a PTN through the same harness. +// Defined once per binary by an engine's host library; drivers such as +// ptn_runner_main.cpp run every engine through the host it returns. std::unique_ptr make_engine_host(); } // namespace ptn::testing diff --git a/backends/native/test/runtime/ggml/BUCK b/backends/native/test/runtime/ggml/BUCK index 2b2add96f39..05de9fa2b72 100644 --- a/backends/native/test/runtime/ggml/BUCK +++ b/backends/native/test/runtime/ggml/BUCK @@ -59,17 +59,32 @@ _VARIANTS = [ }, ] +[ + fbcode_target( + _kind = runtime.cxx_library, + name = "ggml_engine_host" + v["suffix"], + srcs = ["ggml_ptn_runner.cpp"], + link_whole = True, + visibility = ["//executorch/backends/native/test/..."], + exported_deps = ["//executorch/backends/native/test/runtime:engine_host"], + deps = ["//executorch/backends/native/runtime/engine:" + v["engine"]] + + v.get("deps", []), + external_deps = ["gflags"] + v.get("external_deps", []), + labels = v.get("labels", []), + **v.get("compat", {}) + ) + for v in _VARIANTS +] + [ fbcode_target( _kind = runtime.cxx_binary, name = "ggml_ptn_runner" + v["suffix"], - srcs = ["ggml_ptn_runner.cpp"], visibility = ["//executorch/backends/native/test/..."], deps = [ - "//executorch/backends/native/runtime/engine:" + v["engine"], + ":ggml_engine_host" + v["suffix"], "//executorch/backends/native/test/runtime:ptn_runner_main", - ] + v.get("deps", []), - external_deps = ["gflags"] + v.get("external_deps", []), + ], labels = v.get("labels", []), **v.get("compat", {}) ) diff --git a/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp b/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp index 6b725ee46f4..85336504930 100644 --- a/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp +++ b/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp @@ -4,7 +4,7 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -// The ggml engine for ptn_runner_main.cpp. +// The ggml engine host for PTN test drivers. #include diff --git a/backends/native/test/runtime/vulkan/targets.bzl b/backends/native/test/runtime/vulkan/targets.bzl index afb7902c023..1e707ee89c0 100644 --- a/backends/native/test/runtime/vulkan/targets.bzl +++ b/backends/native/test/runtime/vulkan/targets.bzl @@ -41,12 +41,20 @@ def define_common_targets(): cmd = "mkdir -p $OUT && cp $(location fbsource//third-party/swiftshader/lib/linux-x64:libvk_swiftshader_so)/libvulkan.so.1 $OUT/libvk_swiftshader.so", ) + runtime.cxx_library( + name = "vulkan_engine_host", + srcs = ["vulkan_ptn_runner.cpp"], + link_whole = True, + visibility = ["//executorch/backends/native/test/..."], + exported_deps = ["//executorch/backends/native/test/runtime:engine_host"], + deps = ["//executorch/backends/native/runtime/vulkan:vulkan_engine"], + ) + runtime.cxx_binary( name = "vulkan_ptn_runner", - srcs = ["vulkan_ptn_runner.cpp"], visibility = ["//executorch/backends/native/test/..."], deps = [ - "//executorch/backends/native/runtime/vulkan:vulkan_engine", + ":vulkan_engine_host", "//executorch/backends/native/test/runtime:ptn_runner_main", "fbsource//third-party/swiftshader/lib/linux-x64:libvk_swiftshader_fbcode", ], diff --git a/backends/native/test/runtime/vulkan/vulkan_ptn_runner.cpp b/backends/native/test/runtime/vulkan/vulkan_ptn_runner.cpp index b4695b6a784..3f8b9e76ee0 100644 --- a/backends/native/test/runtime/vulkan/vulkan_ptn_runner.cpp +++ b/backends/native/test/runtime/vulkan/vulkan_ptn_runner.cpp @@ -4,7 +4,7 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. -// The Vulkan engine for ptn_runner_main.cpp. +// The Vulkan engine host for PTN test drivers. #include #include From b91949b3942732536816d98dea09b839da58deb2 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:22 -0800 Subject: [PATCH 03/20] [executorch][native] Record op packages with bundled golden and precise samples Pull Request resolved: https://github.com/pytorch/executorch/pull/24357 Engines need op-level reference data that doesn't depend on any engine or device. `op_samples` writes one .ptn per op and quantization scheme, bundling `golden` (eager) and `precise` (float64, weights dequantized, activations unquantized) samples over the same inputs. Linear is the first op. Schemes are float, int4wo g32/g128 and 8da4w g32/g128, each with f32, f16 and bf16 methods; int8 weight-only is left out because no native engine runs it. Linear cases are the backend test suite's models, reused through a stand-in runner, plus decode, prefill, large-K, bias-free leading-batch, outlier-channel and zero-row cases. With CMake, the `native_op_samples` target writes them; it is not in ALL, so a plain build never needs torch. Ops and schemes are independent axes. An op is an `Op` in `ops()`, a function returning `Case`s plus the dtypes to record them in; a scheme is a `Scheme` in `schemes()` that decides which cases it fits, quantizes a model and decodes its weights. `generate` records every op under every scheme, one spawned process per package, and writes no package when a scheme fits none of an op's cases. `cases.py` holds the case type and the backend test suite adapter, `schemes.py` the quantization, and `record.py` the packaging, which knows neither torchao nor any op. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657757 @exported-using-ghexport Differential Revision: [D128254877](https://our.internmc.facebook.com/intern/diff/D128254877/) --- backends/native/test/CMakeLists.txt | 18 ++++ backends/native/test/samples/BUCK | 50 +++++++++++ backends/native/test/samples/cases.py | 87 +++++++++++++++++++ backends/native/test/samples/generate.py | 50 +++++++++++ backends/native/test/samples/linear.py | 36 ++++++++ backends/native/test/samples/ops.py | 24 ++++++ backends/native/test/samples/record.py | 92 +++++++++++++++++++++ backends/native/test/samples/schemes.py | 91 ++++++++++++++++++++ backends/native/test/samples/test_record.py | 90 ++++++++++++++++++++ 9 files changed, 538 insertions(+) create mode 100644 backends/native/test/samples/BUCK create mode 100644 backends/native/test/samples/cases.py create mode 100644 backends/native/test/samples/generate.py create mode 100644 backends/native/test/samples/linear.py create mode 100644 backends/native/test/samples/ops.py create mode 100644 backends/native/test/samples/record.py create mode 100644 backends/native/test/samples/schemes.py create mode 100644 backends/native/test/samples/test_record.py diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index 73ea13f321c..eb404c4cba9 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -29,6 +29,24 @@ target_compile_options( $<$>:-fexceptions> ) +# Op packages with bundled samples, one per op and quantization scheme. +set(_native_op_samples ${CMAKE_CURRENT_BINARY_DIR}/op_samples) +file(GLOB _native_op_samples_srcs CONFIGURE_DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/samples/*.py +) +add_custom_command( + OUTPUT ${_native_op_samples}.stamp + COMMAND + ${CMAKE_COMMAND} -E env "PYTHONPATH=${EXECUTORCH_ROOT}/src" + ${PYTHON_EXECUTABLE} -m executorch.backends.native.test.samples.generate + --out ${_native_op_samples} + COMMAND ${CMAKE_COMMAND} -E touch ${_native_op_samples}.stamp + DEPENDS ${_native_op_samples_srcs} + ${EXECUTORCH_ROOT}/backends/test/suite/operators/test_linear.py + VERBATIM +) +add_custom_target(native_op_samples DEPENDS ${_native_op_samples}.stamp) + if(EXECUTORCH_BUILD_NATIVE_VULKAN) add_executable( native_vulkan_passes_test diff --git a/backends/native/test/samples/BUCK b/backends/native/test/samples/BUCK new file mode 100644 index 00000000000..97b4b66b7db --- /dev/null +++ b/backends/native/test/samples/BUCK @@ -0,0 +1,50 @@ +load("@fbcode_macros//build_defs:build_file_migration.bzl", "fbcode_target") +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +oncall("executorch") + +fbcode_target( + _kind = runtime.python_library, + name = "record", + srcs = [ + "cases.py", + "linear.py", + "ops.py", + "record.py", + "schemes.py", + ], + deps = [ + "//caffe2:torch", + "//executorch/backends/test/suite/operators:test_operators", + "//executorch/exir/native:native", + "//pytorch/ao:torchao", + ], +) + +fbcode_target( + _kind = runtime.python_binary, + name = "generate", + srcs = ["generate.py"], + main_function = "executorch.backends.native.test.samples.generate.main", + deps = [":record"], +) + +fbcode_target( + _kind = runtime.python_test, + name = "test_record", + srcs = ["test_record.py"], + deps = [ + ":record", + "//caffe2:torch", + "//executorch/exir/native:native", + ], +) + +# Op packages with bundled samples, one per op and quantization scheme. +fbcode_target( + _kind = runtime.genrule, + name = "op_samples", + out = "op_samples", + cmd = "$(exe :generate) --out $OUT", + visibility = ["//executorch/backends/native/test/..."], +) diff --git a/backends/native/test/samples/cases.py b/backends/native/test/samples/cases.py new file mode 100644 index 00000000000..b08baba963a --- /dev/null +++ b/backends/native/test/samples/cases.py @@ -0,0 +1,87 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Op cases: a model and the inputs of each sample it is recorded with.""" + +import inspect +from dataclasses import dataclass +from types import ModuleType + +import torch + +Inputs = tuple[torch.Tensor, ...] + +# Caps a suite case's input and output elements over its samples, keeping at +# least one sample: a large output alone has enough elements for statistics. +_ELEMENT_BUDGET = 1 << 16 + + +@dataclass +class Case: + name: str + model: torch.nn.Module + # Positional float32 inputs of each sample; the count may vary by case. + inputs: list[Inputs] + + +def outputs(result) -> list[torch.Tensor]: + return list(result) if isinstance(result, (tuple, list)) else [result] + + +def draws(count: int, *shapes: tuple[int, ...]) -> list[Inputs]: + """`count` samples of standard-normal inputs of the given shapes, from a fixed + seed, so equal calls return equal samples.""" + generator = torch.Generator().manual_seed(0) + return [ + tuple(torch.randn(shape, generator=generator) for shape in shapes) + for _ in range(count) + ] + + +class _SuiteRunner: + """Stands in for the backend test suite's runner, keeping what a test lowers.""" + + _flow = None + + def __init__(self) -> None: + self.calls: list[tuple[torch.nn.Module, Inputs]] = [] + + def lower_and_run_model(self, model, inputs, **_) -> None: + self.calls.append((model, tuple(inputs))) + + +def suite_cases(module: ModuleType, samples: int) -> list[Case]: + """One case per model a backend test suite operator module lowers, with + inputs from up to `samples` runs under different seeds.""" + cases = [] + for test_name, test in sorted(vars(module).items()): + if not test_name.startswith("test_"): + continue + runs = [] + for seed in range(samples): + torch.manual_seed(seed) + runner = _SuiteRunner() + if "dtype" in inspect.signature(test).parameters: + test(runner, torch.float32) + else: + test(runner) + runs.append(runner.calls) + lowered = [[(type(m), [x.shape for x in xs]) for m, xs in run] for run in runs] + if any(seed_lowered != lowered[0] for seed_lowered in lowered): + raise ValueError(f"{test_name} lowers different models across seeds") + for i, (model, inputs) in enumerate(runs[0]): + name = f"{test_name.removeprefix('test_')}_{i}" + try: + results = outputs(model(*inputs)) + except RuntimeError as e: + e.add_note(f"while recording suite case {name}") + raise + numel = sum(t.numel() for t in (*inputs, *results)) + count = max(1, min(samples, _ELEMENT_BUDGET // max(1, numel))) + cases.append(Case(name, model, [run[i][1] for run in runs[:count]])) + if not cases: + raise ValueError(f"no test in {module.__name__} lowers a model") + return cases diff --git a/backends/native/test/samples/generate.py b/backends/native/test/samples/generate.py new file mode 100644 index 00000000000..665a17aa22b --- /dev/null +++ b/backends/native/test/samples/generate.py @@ -0,0 +1,50 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Write _.ptn for each op and quantization scheme that fits a case +of the op.""" + +import argparse +import logging +import multiprocessing +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path + +from executorch.backends.native.test.samples.ops import ops +from executorch.backends.native.test.samples.record import record +from executorch.backends.native.test.samples.schemes import schemes + + +def _record(name: str, scheme: str, out: Path) -> None: + logging.basicConfig(level=logging.INFO, force=True) + op = ops()[name] + record(op.cases(), schemes()[scheme], out / f"{name}_{scheme}.ptn", op.dtypes) + + +def main() -> None: + all_ops, all_schemes = ops(), schemes() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--ops", nargs="+", choices=all_ops, default=list(all_ops)) + parser.add_argument( + "--schemes", nargs="+", choices=all_schemes, default=list(all_schemes) + ) + args = parser.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + # Forked workers can't use the CUDA state torch may have initialized here. + spawn = multiprocessing.get_context("spawn") + with ProcessPoolExecutor(mp_context=spawn) as pool: + jobs = [ + pool.submit(_record, name, scheme, args.out) + for name in args.ops + for scheme in args.schemes + ] + for job in jobs: + job.result() + + +if __name__ == "__main__": + main() diff --git a/backends/native/test/samples/linear.py b/backends/native/test/samples/linear.py new file mode 100644 index 00000000000..24c39fe2ccc --- /dev/null +++ b/backends/native/test/samples/linear.py @@ -0,0 +1,36 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Linear cases: the backend test suite's, and shapes and inputs that stress +accumulation and quantization.""" + +import torch +from executorch.backends.native.test.samples.cases import Case, draws, suite_cases +from executorch.backends.test.suite.operators import test_linear + + +def cases() -> list[Case]: + suite = suite_cases(test_linear, samples=4) + torch.manual_seed(0) + # One channel far above the rest, as in LLM activations. + outlier = draws(4, (16, 256)) + for (x,) in outlier: + x[:, 3] *= 32 + zero_row = draws(2, (8, 256)) + for (x,) in zero_row: + x[2] = 0 + return suite + [ + Case("decode", torch.nn.Linear(2048, 512), draws(8, (1, 2048))), + Case("prefill", torch.nn.Linear(1024, 256), draws(2, (64, 1024))), + Case("large_k", torch.nn.Linear(8192, 32), draws(4, (4, 8192))), + Case( + "leading_batch_no_bias", + torch.nn.Linear(256, 96, bias=False), + draws(4, (2, 7, 256)), + ), + Case("outlier_channel", torch.nn.Linear(256, 128), outlier), + Case("zero_row", torch.nn.Linear(256, 64), zero_row), + ] diff --git a/backends/native/test/samples/ops.py b/backends/native/test/samples/ops.py new file mode 100644 index 00000000000..16332b665ee --- /dev/null +++ b/backends/native/test/samples/ops.py @@ -0,0 +1,24 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Recorded ops, keyed by the name their packages are written under.""" + +from collections.abc import Callable +from dataclasses import dataclass + +from executorch.backends.native.test.samples import linear +from executorch.backends.native.test.samples.cases import Case +from executorch.backends.native.test.samples.record import DTYPES + + +@dataclass(frozen=True) +class Op: + cases: Callable[[], list[Case]] + dtypes: tuple[str, ...] = tuple(DTYPES) + + +def ops() -> dict[str, Op]: + return {"linear": Op(linear.cases)} diff --git a/backends/native/test/samples/record.py b/backends/native/test/samples/record.py new file mode 100644 index 00000000000..6846890f081 --- /dev/null +++ b/backends/native/test/samples/record.py @@ -0,0 +1,92 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Record op cases into .ptn packages with bundled samples. + +A package holds one method per case and dtype under one quantization scheme, +and two sample sets over the same inputs: `golden`, the eager outputs, and +`precise`, the same math in float64 with weights dequantized and activations +unquantized, floating outputs stored as float32. Nothing in it depends on an +engine or a device: consumers judge an engine by its distance from `precise` +relative to eager's. +""" + +import copy +import logging +from collections.abc import Iterable +from pathlib import Path + +import torch +from executorch.backends.native.test.samples.cases import Case, Inputs, outputs +from executorch.backends.native.test.samples.schemes import Scheme +from executorch.exir.native import Call, Sample, SampleSet, to_native, write_samples +from torch.func import functional_call + +logger: logging.Logger = logging.getLogger(__name__) + +DTYPES = {"f32": torch.float32, "f16": torch.float16, "bf16": torch.bfloat16} + + +class _Method(torch.nn.ModuleDict): + """One model keyed by its method name, keeping constant FQNs unique across a + package's methods.""" + + def forward(self, *args): + (model,) = self.values() + return model(*args) + + +def _cast(inputs: Inputs, dtype: torch.dtype) -> Inputs: + return tuple(t.to(dtype) if t.is_floating_point() else t for t in inputs) + + +def record( + cases: list[Case], + scheme: Scheme, + path: Path, + dtypes: Iterable[str] = tuple(DTYPES), +) -> None: + """Writes no package, removing any at `path`, when the scheme fits no case.""" + programs, golden, precise = {}, [], [] + for case in cases: + if not scheme.fits(case.model): + logger.info(f"{path.name}: {case.name} does not fit the scheme") + continue + for suffix in dtypes: + name = f"{case.name}_{suffix}" + if name in programs: + raise ValueError(f"{path.name}: duplicate method {name}") + dtype = DTYPES[suffix] + model = copy.deepcopy(case.model).to(dtype).eval() + scheme.quantize(model) + method = _Method({name: model}) + state = { + k: scheme.decode(v).double() for k, v in method.state_dict().items() + } + inputs = [_cast(x, dtype) for x in case.inputs] + programs[name] = torch.export.export(method, inputs[0]) + with torch.no_grad(): + for x in inputs: + golden.append(Sample([Call(name, list(x), outputs(method(*x)))])) + ref = functional_call(method, state, _cast(x, torch.float64)) + ref = [ + t.float() if t.is_floating_point() else t for t in outputs(ref) + ] + precise.append(Sample([Call(name, list(x), ref)])) + if not programs: + logger.info(f"{path.name}: no case fits the scheme, not written") + path.unlink(missing_ok=True) + return + to_native(programs).save(str(path)) + source = f"torch {torch.__version__}" + write_samples(str(path), "golden", SampleSet(golden, "eager", source)) + write_samples( + str(path), + "precise", + SampleSet( + precise, "float64, weights dequantized, activations unquantized", source + ), + ) diff --git a/backends/native/test/samples/schemes.py b/backends/native/test/samples/schemes.py new file mode 100644 index 00000000000..8966e7facad --- /dev/null +++ b/backends/native/test/samples/schemes.py @@ -0,0 +1,91 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Quantization schemes. Every op is recorded under every scheme, keeping the +cases the scheme fits.""" + +from dataclasses import dataclass + +import torch +from torchao.core.config import AOBaseConfig +from torchao.quantization import ( + Int8DynamicActivationIntxWeightConfig, + IntxWeightOnlyConfig, + quantize_, +) +from torchao.quantization.granularity import PerGroup +from torchao.utils import TorchAOBaseTensor + + +class Scheme: + """No quantization.""" + + def fits(self, model: torch.nn.Module) -> bool: + return True + + def quantize(self, model: torch.nn.Module) -> None: + pass + + def decode(self, value: torch.Tensor) -> torch.Tensor: + """A state tensor of a quantized model as a plain tensor.""" + return value + + +def _fits(model: torch.nn.Module, group: int | None) -> bool: + """Whether `model` has an nn.Linear and every one's in_features split into + groups of `group`, or form one group when it is None.""" + linears = [m for m in model.modules() if isinstance(m, torch.nn.Linear)] + for m in linears: + size = m.in_features if group is None else group + # torchao reduces a one-element-group weight to a single scale. + if size < 2 or m.in_features % size: + return False + return bool(linears) + + +@dataclass(frozen=True) +class Torchao(Scheme): + """A torchao config applied to every nn.Linear.""" + + config: AOBaseConfig + + def fits(self, model: torch.nn.Module) -> bool: + granularity = ( + self.config.granularity + if isinstance(self.config, IntxWeightOnlyConfig) + else self.config.weight_granularity + ) + return _fits( + model, granularity.group_size if isinstance(granularity, PerGroup) else None + ) + + def quantize(self, model: torch.nn.Module) -> None: + quantize_(model, self.config) + + def decode(self, value: torch.Tensor) -> torch.Tensor: + return value.dequantize() if isinstance(value, TorchAOBaseTensor) else value + + +def schemes() -> dict[str, Scheme]: + return { + "float": Scheme(), + "int4wo_g32": Torchao( + IntxWeightOnlyConfig(weight_dtype=torch.int4, granularity=PerGroup(32)) + ), + "int4wo_g128": Torchao( + IntxWeightOnlyConfig(weight_dtype=torch.int4, granularity=PerGroup(128)) + ), + "8da4w_g32": Torchao( + Int8DynamicActivationIntxWeightConfig( + weight_dtype=torch.int4, weight_granularity=PerGroup(32) + ) + ), + "8da4w_g128": Torchao( + Int8DynamicActivationIntxWeightConfig( + weight_dtype=torch.int4, weight_granularity=PerGroup(128) + ) + ), + } diff --git a/backends/native/test/samples/test_record.py b/backends/native/test/samples/test_record.py new file mode 100644 index 00000000000..391ecd9b52a --- /dev/null +++ b/backends/native/test/samples/test_record.py @@ -0,0 +1,90 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import tempfile +import unittest +from pathlib import Path +from types import ModuleType + +import torch +import torch.nn.functional as F +from executorch.backends.native.test.samples.cases import Case, draws, suite_cases +from executorch.backends.native.test.samples.record import record +from executorch.backends.native.test.samples.schemes import Scheme, schemes +from executorch.exir.native import read_samples + + +class RecordTest(unittest.TestCase): + def test_suite_cases_raises_on_a_model_eager_rejects(self) -> None: + module = ModuleType("suite") + module.test_mismatch = lambda runner: runner.lower_and_run_model( + torch.nn.Linear(4, 2), (torch.randn(1, 3),) + ) + with self.assertRaises(RuntimeError) as raised: + suite_cases(module, samples=1) + self.assertIn( + "while recording suite case mismatch_0", raised.exception.__notes__ + ) + + def test_suite_cases_raises_when_seeds_lower_different_shapes(self) -> None: + module = ModuleType("suite") + sizes = iter([2, 3]) + module.test_varying = lambda runner: runner.lower_and_run_model( + torch.nn.Identity(), (torch.randn(next(sizes)),) + ) + with self.assertRaises(ValueError): + suite_cases(module, samples=2) + + def test_suite_cases_raises_when_no_test_lowers_a_model(self) -> None: + with self.assertRaises(ValueError): + suite_cases(ModuleType("suite"), samples=1) + + def test_record_raises_on_duplicate_case_names(self) -> None: + case = Case("same", torch.nn.Linear(2, 2), draws(1, (1, 2))) + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(ValueError): + record([case, case], Scheme(), Path(tmp) / "dup.ptn") + + def test_record_writes_no_package_when_the_scheme_fits_no_case(self) -> None: + case = Case("relu", torch.nn.ReLU(), draws(1, (1, 32))) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "relu.ptn" + path.write_bytes(b"stale") + record([case], schemes()["int4wo_g32"], path) + self.assertFalse(path.exists()) + + def test_record_golden_is_eager_and_precise_is_float64(self) -> None: + model = torch.nn.Linear(16, 4) + (x,) = draws(1, (3, 16))[0] + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "linear.ptn" + record([Case("linear", model, [(x,)])], Scheme(), path, dtypes=["bf16"]) + sets = read_samples(str(path)) + x, w, b = (t.detach().bfloat16() for t in (x, model.weight, model.bias)) + torch.testing.assert_close( + sets["golden"].samples[0].calls[0].outputs[0], + F.linear(x, w, b), + rtol=0, + atol=0, + ) + torch.testing.assert_close( + sets["precise"].samples[0].calls[0].outputs[0], + F.linear(x.double(), w.double(), b.double()).float(), + rtol=0, + atol=0, + ) + + def test_record_keeps_integer_outputs_precise(self) -> None: + class Argmax(torch.nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return x.argmax(-1) + + case = Case("argmax", Argmax(), draws(1, (2, 8))) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "argmax.ptn" + record([case], Scheme(), path, dtypes=["f32"]) + precise = read_samples(str(path))["precise"].samples + self.assertEqual(precise[0].calls[0].outputs[0].dtype, torch.int64) From 0e7dd3139aa4d72f87949097ddb419a32d810bbf Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:23 -0800 Subject: [PATCH 04/20] [executorch][native] Pin eager kernels when generating op samples Pull Request resolved: https://github.com/pytorch/executorch/pull/24358 Eager golden outputs differed by an ulp or two between generating hosts, because MKL and oneDNN pick kernels by ISA and MKL splits work by thread count. The `op_samples` genrule and the `native_op_samples` CMake target now pin MKL to its reproducible path, oneDNN and ATen to AVX2, and OpenMP and MKL to one thread. `MKL_NUM_THREADS` is set as well as `OMP_NUM_THREADS` so a host's own value cannot override it. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657753 @exported-using-ghexport Differential Revision: [D128254894](https://our.internmc.facebook.com/intern/diff/D128254894/) --- backends/native/test/CMakeLists.txt | 11 +++++++---- backends/native/test/samples/BUCK | 6 ++++-- backends/native/test/samples/generate.py | 7 ++++++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index eb404c4cba9..4e9c7b28837 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -29,7 +29,9 @@ target_compile_options( $<$>:-fexceptions> ) -# Op packages with bundled samples, one per op and quantization scheme. +# Op packages with bundled samples, one per op and quantization scheme. The +# environment pins the eager kernels' ISA and threads, so the golden is the same +# on any x86 host with AVX2. Other hosts can differ. set(_native_op_samples ${CMAKE_CURRENT_BINARY_DIR}/op_samples) file(GLOB _native_op_samples_srcs CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/samples/*.py @@ -37,9 +39,10 @@ file(GLOB _native_op_samples_srcs CONFIGURE_DEPENDS add_custom_command( OUTPUT ${_native_op_samples}.stamp COMMAND - ${CMAKE_COMMAND} -E env "PYTHONPATH=${EXECUTORCH_ROOT}/src" - ${PYTHON_EXECUTABLE} -m executorch.backends.native.test.samples.generate - --out ${_native_op_samples} + ${CMAKE_COMMAND} -E env MKL_CBWR=COMPATIBLE ONEDNN_MAX_CPU_ISA=AVX2 + ATEN_CPU_CAPABILITY=avx2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 + "PYTHONPATH=${EXECUTORCH_ROOT}/src" ${PYTHON_EXECUTABLE} -m + executorch.backends.native.test.samples.generate --out ${_native_op_samples} COMMAND ${CMAKE_COMMAND} -E touch ${_native_op_samples}.stamp DEPENDS ${_native_op_samples_srcs} ${EXECUTORCH_ROOT}/backends/test/suite/operators/test_linear.py diff --git a/backends/native/test/samples/BUCK b/backends/native/test/samples/BUCK index 97b4b66b7db..48e83efc2d9 100644 --- a/backends/native/test/samples/BUCK +++ b/backends/native/test/samples/BUCK @@ -40,11 +40,13 @@ fbcode_target( ], ) -# Op packages with bundled samples, one per op and quantization scheme. +# Op packages with bundled samples, one per op and quantization scheme. The +# environment pins the eager kernels' ISA and threads, so the golden is the same +# on any x86 host with AVX2. Other hosts can differ. fbcode_target( _kind = runtime.genrule, name = "op_samples", out = "op_samples", - cmd = "$(exe :generate) --out $OUT", + cmd = "MKL_CBWR=COMPATIBLE ONEDNN_MAX_CPU_ISA=AVX2 ATEN_CPU_CAPABILITY=avx2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 $(exe :generate) --out $OUT", visibility = ["//executorch/backends/native/test/..."], ) diff --git a/backends/native/test/samples/generate.py b/backends/native/test/samples/generate.py index 665a17aa22b..f98fe7e91cb 100644 --- a/backends/native/test/samples/generate.py +++ b/backends/native/test/samples/generate.py @@ -5,7 +5,12 @@ # LICENSE file in the root directory of this source tree. """Write _.ptn for each op and quantization scheme that fits a case -of the op.""" +of the op. + +Golden outputs depend on the eager kernels' ISA and threads; the op_samples +genrule and the native_op_samples CMake target pin them, so build one of those +rather than running this module to reproduce their packages. +""" import argparse import logging From 7b4699be7c983d49fabe836dfc07f7e9edc63e6e Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:24 -0800 Subject: [PATCH 05/20] [executorch][native] Judge engine outputs against golden and precise samples Pull Request resolved: https://github.com/pytorch/executorch/pull/24359 A fixed tolerance either hides engine bugs or fails legitimate numerics, so an engine is judged by its distance from `precise` relative to eager's. `OutputCheck` bounds RMSE and max error by multiples of eager's plus `rtol`, and the fitted absolute and relative bias likewise, plus `sigmas` standard errors. Non-finite values must equal golden. CMake builds it and its test too. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657743 @exported-using-ghexport Differential Revision: [D128254878](https://our.internmc.facebook.com/intern/diff/D128254878/) --- backends/native/test/CMakeLists.txt | 15 + backends/native/test/samples/BUCK | 16 ++ backends/native/test/samples/OutputCheck.cpp | 198 ++++++++++++++ backends/native/test/samples/OutputCheck.h | 93 +++++++ .../native/test/samples/test_output_check.cpp | 257 ++++++++++++++++++ 5 files changed, 579 insertions(+) create mode 100644 backends/native/test/samples/OutputCheck.cpp create mode 100644 backends/native/test/samples/OutputCheck.h create mode 100644 backends/native/test/samples/test_output_check.cpp diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index 4e9c7b28837..f6a72ffde51 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -29,6 +29,21 @@ target_compile_options( $<$>:-fexceptions> ) +# Judges an engine's outputs against a package's golden and precise samples. +add_library(native_output_check STATIC samples/OutputCheck.cpp) +target_link_libraries(native_output_check PUBLIC native_package) +add_executable(native_output_check_test samples/test_output_check.cpp) +target_link_libraries( + native_output_check_test PRIVATE native_output_check GTest::gtest_main +) +add_test(NAME native_output_check_test COMMAND native_output_check_test) +foreach(target native_output_check native_output_check_test) + target_compile_options( + ${target} PRIVATE $<$:/EHsc> + $<$>:-fexceptions> + ) +endforeach() + # Op packages with bundled samples, one per op and quantization scheme. The # environment pins the eager kernels' ISA and threads, so the golden is the same # on any x86 host with AVX2. Other hosts can differ. diff --git a/backends/native/test/samples/BUCK b/backends/native/test/samples/BUCK index 48e83efc2d9..a59019b033c 100644 --- a/backends/native/test/samples/BUCK +++ b/backends/native/test/samples/BUCK @@ -50,3 +50,19 @@ fbcode_target( cmd = "MKL_CBWR=COMPATIBLE ONEDNN_MAX_CPU_ISA=AVX2 ATEN_CPU_CAPABILITY=avx2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 $(exe :generate) --out $OUT", visibility = ["//executorch/backends/native/test/..."], ) + +# Judges an engine's outputs against a package's golden and precise samples. +fbcode_target( + _kind = runtime.cxx_library, + name = "output_check", + srcs = ["OutputCheck.cpp"], + exported_headers = ["OutputCheck.h"], + exported_deps = ["//executorch/backends/native/runtime/deserialize:samples"], +) + +fbcode_target( + _kind = runtime.cxx_test, + name = "output_check_test", + srcs = ["test_output_check.cpp"], + deps = [":output_check"], +) diff --git a/backends/native/test/samples/OutputCheck.cpp b/backends/native/test/samples/OutputCheck.cpp new file mode 100644 index 00000000000..f60611a847f --- /dev/null +++ b/backends/native/test/samples/OutputCheck.cpp @@ -0,0 +1,198 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace ptn::testing { +namespace { + +uint32_t load_u16(const uint8_t* bytes) { + return static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8); +} + +float half_to_float(const uint8_t* bytes) { + const uint32_t h = load_u16(bytes); + const uint32_t sign = (h & 0x8000u) << 16; + const uint32_t exponent = (h >> 10) & 0x1Fu; + const uint32_t mantissa = h & 0x3FFu; + if (exponent == 0) { + const float magnitude = std::ldexp(static_cast(mantissa), -24); + return sign != 0 ? -magnitude : magnitude; + } + if (exponent == 0x1Fu) { + return std::bit_cast(sign | 0x7F800000u | (mantissa << 13)); + } + return std::bit_cast( + sign | ((exponent + 112) << 23) | (mantissa << 13)); +} + +bool same(double a, double b) { + return a == b || (std::isnan(a) && std::isnan(b)); +} + +std::string g3(double v) { + char text[32]; + std::snprintf(text, sizeof(text), "%.3g", v); + return text; +} + +} // namespace + +std::vector to_float(const SampleTensor& t) { + const size_t numel = t.bytes.size() / element_size(t.dtype); + std::vector out(numel); + const uint8_t* bytes = t.bytes.data(); + switch (t.dtype) { + case kFloat: + std::memcpy(out.data(), bytes, numel * sizeof(float)); + break; + case kDouble: + for (size_t i = 0; i < numel; ++i) { + double v; + std::memcpy(&v, bytes + i * sizeof(double), sizeof(double)); + out[i] = static_cast(v); + } + break; + case kHalf: + for (size_t i = 0; i < numel; ++i) { + out[i] = half_to_float(bytes + 2 * i); + } + break; + case kBFloat16: + for (size_t i = 0; i < numel; ++i) { + out[i] = std::bit_cast(load_u16(bytes + 2 * i) << 16); + } + break; + case kByte: + case kChar: + case kShort: + case kInt: + case kLong: + case kBool: + case kUInt16: + case kUInt32: + case kUInt64: + throw std::invalid_argument( + std::string("to_float: unsupported dtype ") + + scalar_type_name(t.dtype)); + } + return out; +} + +// Welford co-moments, stable when the means are large against the spread. +void OutputCheck::Fit::add(double p, double d) { + ++n; + const double dp = p - mean_p; + const double dd = d - mean_d; + mean_p += dp / n; + mean_d += dd / n; + cpp += dp * (p - mean_p); + cpd += dp * (d - mean_d); + cdd += dd * (d - mean_d); + sum_dd += d * d; + max_abs = std::max(max_abs, std::abs(d)); +} + +ErrorStats OutputCheck::Fit::stats() const { + ErrorStats s{.count = n, .max_abs = max_abs}; + if (n == 0) { + return s; + } + const bool sloped = cpp > 0.0; + s.rmse = std::sqrt(sum_dd / n); + s.rel_bias = sloped ? cpd / cpp : 0.0; + s.abs_bias = mean_d - s.rel_bias * mean_p; + const int64_t dof = n - (sloped ? 2 : 1); + const double residual = dof > 0 ? std::max(0.0, cdd - s.rel_bias * cpd) / dof + : std::numeric_limits::infinity(); + s.abs_bias_se = + std::sqrt(residual * (1.0 / n + (sloped ? mean_p * mean_p / cpp : 0.0))); + s.rel_bias_se = sloped ? std::sqrt(residual / cpp) : 0.0; + return s; +} + +void OutputCheck::add( + std::span got, + const SampleTensor& golden, + const SampleTensor& precise) { + const std::vector want = to_float(golden); + const std::vector ref = to_float(precise); + if (got.size() != want.size() || got.size() != ref.size()) { + throw std::invalid_argument( + "OutputCheck: " + std::to_string(got.size()) + " values against " + + std::to_string(want.size()) + " golden and " + + std::to_string(ref.size()) + " precise"); + } + for (size_t i = 0; i < got.size(); ++i) { + const double e = got[i]; + const double g = want[i]; + const double p = ref[i]; + if (!std::isfinite(e) || !std::isfinite(g) || !std::isfinite(p)) { + if (!same(e, g)) { + ++nonfinite_mismatches_; + } + continue; + } + engine_.add(p, e - p); + eager_.add(p, g - p); + sum_pp_ += p * p; + max_p_ = std::max(max_p_, std::abs(p)); + } +} + +std::vector OutputCheck::failures(const Policy& policy) const { + const ErrorStats e = engine(); + const ErrorStats g = eager(); + const double rms_p = e.count > 0 ? std::sqrt(sum_pp_ / e.count) : 0.0; + std::vector out; + const auto bound = [&](const char* name, double value, double limit) { + if (!(value <= limit)) { + out.push_back(std::string(name) + ": " + g3(value) + " > " + g3(limit)); + } + }; + bound("rmse", e.rmse, policy.rmse_ratio * g.rmse + policy.rtol * rms_p); + bound( + "max_abs", + e.max_abs, + policy.max_ratio * g.max_abs + policy.rtol * max_p_); + bound( + "abs_bias", + std::abs(e.abs_bias), + policy.rmse_ratio * std::abs(g.abs_bias) + policy.rtol * rms_p + + policy.sigmas * e.abs_bias_se); + bound( + "rel_bias", + std::abs(e.rel_bias), + policy.rmse_ratio * std::abs(g.rel_bias) + policy.rtol + + policy.sigmas * e.rel_bias_se); + if (nonfinite_mismatches_ > 0) { + out.push_back( + "nonfinite: " + std::to_string(nonfinite_mismatches_) + + " values differ from golden where a value is NaN or infinite"); + } + return out; +} + +std::string OutputCheck::summary() const { + const ErrorStats e = engine(); + const ErrorStats g = eager(); + return "n " + std::to_string(e.count) + ", engine (eager): rmse " + + g3(e.rmse) + " (" + g3(g.rmse) + "), max_abs " + g3(e.max_abs) + " (" + + g3(g.max_abs) + "), abs_bias " + g3(e.abs_bias) + " +-" + + g3(e.abs_bias_se) + " (" + g3(g.abs_bias) + "), rel_bias " + + g3(e.rel_bias) + " +-" + g3(e.rel_bias_se) + " (" + g3(g.rel_bias) + ")"; +} + +} // namespace ptn::testing diff --git a/backends/native/test/samples/OutputCheck.h b/backends/native/test/samples/OutputCheck.h new file mode 100644 index 00000000000..957cca691c3 --- /dev/null +++ b/backends/native/test/samples/OutputCheck.h @@ -0,0 +1,93 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include +#include + +#include + +namespace ptn::testing { + +// Values of a Float, Half, BFloat16 or Double sample tensor. Throws +// std::invalid_argument for any other dtype. +std::vector to_float(const SampleTensor& t); + +// Error d = x - precise, fitted by least squares as +// d = abs_bias + rel_bias * precise; the _se fields are the fit's standard +// errors. +struct ErrorStats { + int64_t count = 0; + double rmse = 0.0; + double max_abs = 0.0; + double abs_bias = 0.0; + double abs_bias_se = 0.0; + double rel_bias = 0.0; + double rel_bias_se = 0.0; +}; + +// Bounds on an engine's error statistics: eager's times rmse_ratio (max_ratio +// for max_abs), plus rtol times the precise outputs' RMS (their max for +// max_abs, 1 for rel_bias). The fitted biases may also deviate by `sigmas` +// standard errors. +struct Policy { + double rmse_ratio = 3.0; + double max_ratio = 4.0; + double rtol = 1e-5; + double sigmas = 6.0; +}; + +// One output of a method, accumulated over its samples. Statistics cover the +// positions where the engine, golden and precise values are all finite; +// elsewhere the engine must equal golden, NaN matching NaN. +class OutputCheck { + public: + // Throws std::invalid_argument when the element counts differ. + void add( + std::span got, + const SampleTensor& golden, + const SampleTensor& precise); + + ErrorStats engine() const { + return engine_.stats(); + } + ErrorStats eager() const { + return eager_.stats(); + } + + // One ": " message per violated rule, among rmse, max_abs, + // abs_bias, rel_bias and nonfinite; empty when the engine passes. + std::vector failures(const Policy& policy) const; + + std::string summary() const; + + private: + // Streaming least-squares fit of d against p. + struct Fit { + int64_t n = 0; + double mean_p = 0.0; + double mean_d = 0.0; + double cpp = 0.0; + double cpd = 0.0; + double cdd = 0.0; + double sum_dd = 0.0; + double max_abs = 0.0; + + void add(double p, double d); + ErrorStats stats() const; + }; + + Fit engine_; + Fit eager_; + double sum_pp_ = 0.0; + double max_p_ = 0.0; + int64_t nonfinite_mismatches_ = 0; +}; + +} // namespace ptn::testing diff --git a/backends/native/test/samples/test_output_check.cpp b/backends/native/test/samples/test_output_check.cpp new file mode 100644 index 00000000000..73e3d122ee5 --- /dev/null +++ b/backends/native/test/samples/test_output_check.cpp @@ -0,0 +1,257 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace ptn::testing { +namespace { + +constexpr size_t kCount = 4096; +constexpr float kInf = std::numeric_limits::infinity(); +constexpr float kNaN = std::numeric_limits::quiet_NaN(); + +SampleTensor f32(const std::vector& values) { + SampleTensor t{ + .dtype = kFloat, .sizes = {static_cast(values.size())}}; + t.bytes.resize(values.size() * sizeof(float)); + std::memcpy(t.bytes.data(), values.data(), t.bytes.size()); + return t; +} + +SampleTensor bits16(ScalarType dtype, const std::vector& bits) { + SampleTensor t{.dtype = dtype, .sizes = {static_cast(bits.size())}}; + for (const uint16_t b : bits) { + t.bytes.push_back(b & 0xFF); + t.bytes.push_back(b >> 8); + } + return t; +} + +std::vector normal(double stddev, uint32_t seed) { + std::mt19937 rng(seed); + std::normal_distribution dist(0.0, stddev); + std::vector out(kCount); + for (float& v : out) { + v = static_cast(dist(rng)); + } + return out; +} + +// x * scale + noise + bias, elementwise. +std::vector affine( + const std::vector& x, + double scale, + const std::vector& noise, + double bias = 0.0) { + std::vector out(x.size()); + for (size_t i = 0; i < x.size(); ++i) { + out[i] = static_cast(x[i] * scale + noise[i] + bias); + } + return out; +} + +std::vector failed_rules( + const std::vector& got, + const std::vector& golden, + const std::vector& precise, + const Policy& policy = {}) { + OutputCheck check; + check.add(got, f32(golden), f32(precise)); + std::vector rules; + for (const std::string& failure : check.failures(policy)) { + rules.push_back(failure.substr(0, failure.find(':'))); + } + return rules; +} + +class OutputCheckTest : public ::testing::Test { + protected: + const std::vector precise_ = normal(1.0, 0); + const std::vector golden_ = affine(precise_, 1.0, normal(1e-3, 1)); + const std::vector engine_noise_ = normal(1e-3, 2); +}; + +// cppcheck-suppress-begin syntaxError +TEST(ToFloatTest, DecodesHalfAndBFloat16) { + const std::vector half = {1.5f, -2.0f, std::ldexp(1.0f, -24), -kInf}; + EXPECT_EQ(to_float(bits16(kHalf, {0x3E00, 0xC000, 0x0001, 0xFC00})), half); + const std::vector bf16 = {1.5f, -2.0f, kInf}; + EXPECT_EQ(to_float(bits16(kBFloat16, {0x3FC0, 0xC000, 0x7F80})), bf16); +} + +TEST(ToFloatTest, NarrowsDouble) { + const std::vector values = {1.5, -2.0, 0.1}; + SampleTensor t{.dtype = kDouble, .sizes = {3}}; + t.bytes.resize(values.size() * sizeof(double)); + std::memcpy(t.bytes.data(), values.data(), t.bytes.size()); + const std::vector expected = {1.5f, -2.0f, 0.1f}; + EXPECT_EQ(to_float(t), expected); +} + +TEST(ToFloatTest, RejectsIntegers) { + EXPECT_THROW(to_float(bits16(kShort, {1})), std::invalid_argument); +} + +TEST_F(OutputCheckTest, PassesNoiseLikeEagers) { + EXPECT_EQ( + failed_rules(affine(precise_, 1.0, engine_noise_), golden_, precise_), + std::vector{}); +} + +TEST_F(OutputCheckTest, PassesUlpErrorsAgainstAnExactEager) { + std::vector got = precise_; + for (float& v : got) { + v = std::nextafter(v, kInf); + } + EXPECT_EQ(failed_rules(got, precise_, precise_), std::vector{}); +} + +TEST_F(OutputCheckTest, PassesBiasesSharedWithEager) { + const std::vector golden = + affine(precise_, 1.002, normal(1e-3, 1), 1e-3); + EXPECT_EQ( + failed_rules( + affine(precise_, 1.002, engine_noise_, 1e-3), golden, precise_), + std::vector{}); +} + +TEST_F(OutputCheckTest, FailsNoiseFarAboveEagers) { + const std::vector expected = {"rmse", "max_abs"}; + EXPECT_EQ( + failed_rules(affine(precise_, 1.0, normal(1e-2, 2)), golden_, precise_), + expected); +} + +TEST_F(OutputCheckTest, PassesNoiseFarAboveEagersUnderLooserRatios) { + EXPECT_EQ( + failed_rules( + affine(precise_, 1.0, normal(1e-2, 2)), + golden_, + precise_, + Policy{.rmse_ratio = 20.0, .max_ratio = 20.0}), + std::vector{}); +} + +TEST_F(OutputCheckTest, FailsOneOutlier) { + std::vector precise = precise_; + std::vector golden = golden_; + precise[7] = golden[7] = 0.0f; + std::vector got = affine(precise, 1.0, engine_noise_); + got[7] = 0.1f; + EXPECT_EQ( + failed_rules(got, golden, precise), std::vector{"max_abs"}); +} + +TEST_F(OutputCheckTest, FailsSmallAbsoluteBias) { + EXPECT_EQ( + failed_rules( + affine(precise_, 1.0, engine_noise_, 1e-3), golden_, precise_), + std::vector{"abs_bias"}); +} + +TEST_F(OutputCheckTest, FailsSmallRelativeBias) { + EXPECT_EQ( + failed_rules(affine(precise_, 1.002, engine_noise_), golden_, precise_), + std::vector{"rel_bias"}); +} + +TEST_F(OutputCheckTest, ExcludesNonFiniteMatchingGolden) { + std::vector precise = precise_; + std::vector golden = golden_; + std::vector got = affine(precise_, 1.0, engine_noise_); + golden[0] = got[0] = kNaN; + golden[1] = got[1] = -kInf; + precise[2] = kInf; + got[2] = golden[2]; + + OutputCheck check; + check.add(got, f32(golden), f32(precise)); + + EXPECT_EQ(check.failures(Policy{}), std::vector{}); + EXPECT_EQ(check.engine().count, kCount - 3); +} + +TEST_F(OutputCheckTest, FailsNonFiniteDifferingFromGolden) { + std::vector precise = precise_; + std::vector golden = golden_; + std::vector got = affine(precise_, 1.0, engine_noise_); + golden[0] = kInf; + got[0] = -kInf; + got[1] = kNaN; + golden[2] = kNaN; + precise[3] = kNaN; + + OutputCheck check; + check.add(got, f32(golden), f32(precise)); + + const std::vector expected = { + "nonfinite: 4 values differ from golden where a value is NaN or infinite"}; + EXPECT_EQ(check.failures(Policy{}), expected); +} + +TEST_F(OutputCheckTest, AccumulatesAcrossSamples) { + const std::vector got = affine(precise_, 1.0, engine_noise_, 1e-3); + const size_t half = kCount / 2; + const auto part = [&](const std::vector& v, size_t begin) { + return std::vector(v.begin() + begin, v.begin() + begin + half); + }; + + OutputCheck whole; + whole.add(got, f32(golden_), f32(precise_)); + OutputCheck split; + for (const size_t begin : {size_t{0}, half}) { + split.add( + part(got, begin), + f32(part(golden_, begin)), + f32(part(precise_, begin))); + } + + EXPECT_EQ(split.engine().count, whole.engine().count); + EXPECT_DOUBLE_EQ(split.engine().abs_bias, whole.engine().abs_bias); + EXPECT_DOUBLE_EQ(split.engine().rel_bias_se, whole.engine().rel_bias_se); +} + +TEST(OutputCheckFitTest, MatchesClosedFormOnAnOffsetPrecise) { + // d = got - precise = 1 + 0.5 * precise + e, with e orthogonal to 1 and to + // precise, so the fit is exact and the residual variance is |e|^2 / (n - 2). + const std::vector precise = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f}; + const std::vector got = {2.5f, 4.25f, 5.0f, 7.25f, 8.5f}; + OutputCheck check; + check.add(got, f32(precise), f32(precise)); + + const ErrorStats stats = check.engine(); + const double residual = 0.375 / 3; + EXPECT_EQ(stats.count, 5); + EXPECT_NEAR(stats.rmse, std::sqrt(34.125 / 5), 1e-12); + EXPECT_EQ(stats.max_abs, 3.5); + EXPECT_NEAR(stats.abs_bias, 1.0, 1e-12); + EXPECT_NEAR(stats.rel_bias, 0.5, 1e-12); + EXPECT_NEAR( + stats.abs_bias_se, std::sqrt(residual * (1.0 / 5 + 9.0 / 10)), 1e-12); + EXPECT_NEAR(stats.rel_bias_se, std::sqrt(residual / 10), 1e-12); +} + +TEST(OutputCheckSizeTest, RejectsMismatchedCounts) { + OutputCheck check; + EXPECT_THROW( + check.add(std::vector{1.0f}, f32({1.0f, 2.0f}), f32({1.0f})), + std::invalid_argument); +} +// cppcheck-suppress-end syntaxError + +} // namespace +} // namespace ptn::testing From 0acda5755a7dcffe1ef4cc8e643fbb60f4009cd3 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:25 -0800 Subject: [PATCH 06/20] [executorch][native] Run engines on bundled op samples with gtest Pull Request resolved: https://github.com/pytorch/executorch/pull/24360 The bundled samples need a harness that runs on any engine. `sample_test_main` registers one gtest per package and method and judges each output with `OutputCheck`; unsupported ops, dtypes and shapes fail. `ggml_sample_test{,_cuda,_hip}` and `vulkan_sample_test` run it on `op_samples`. With CMake, `native_ggml_sample_test` (CPU) and `native_vulkan_sample_test` depend on `native_op_samples` and run it on its output. The sample tests are manual for now, since gaps fail: `ci.skip_test` in Buck and `DISABLED` in CTest. The README says how to run them and what the generator needs. `vulkan_sample_test` runs on SwiftShader like `vulkan_engine_test`; preloading a driver's `libvulkan.so.1` runs it on that driver. The ggml CUDA targets preload the CUDA driver stub through their env, so `test_ggml_ops.py` no longer puts the runner's directory on `LD_LIBRARY_PATH`. Authored with Claude Code. Differential Revision: [D128254880](https://our.internmc.facebook.com/intern/diff/D128254880/) **NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D128254880/)! ghstack-source-id: 463657748 --- backends/native/README.md | 7 +- backends/native/runtime/deserialize/Samples.h | 2 + backends/native/test/CMakeLists.txt | 59 ++++ backends/native/test/runtime/ggml/BUCK | 34 ++- .../native/test/runtime/ggml/test_ggml_ops.py | 6 - .../native/test/runtime/vulkan/targets.bzl | 17 ++ backends/native/test/samples/BUCK | 22 ++ .../native/test/samples/sample_test_main.cpp | 251 ++++++++++++++++++ 8 files changed, 383 insertions(+), 15 deletions(-) create mode 100644 backends/native/test/samples/sample_test_main.cpp diff --git a/backends/native/README.md b/backends/native/README.md index 9ce1b69d277..8c51b12efac 100644 --- a/backends/native/README.md +++ b/backends/native/README.md @@ -206,7 +206,12 @@ and API. `native_cpu_engine_test` covers the CPU engine when `EXECUTORCH_BUILD_NATIVE_CPU` is enabled. `native_vulkan_passes_test` covers the Vulkan graph rewrites without requiring a GPU. When either runner below is enabled, `native_runner_utils_test` covers their shared runner utilities. Run these -with `ctest --test-dir cmake-out -R native_`. +with `ctest --test-dir cmake-out -R native_` after `cmake --build cmake-out`. + +The engines' sample tests judge them on the bundled op samples. Building them runs +the sample generator, which needs `torch` and `torchao`. Those with failing cases +are disabled in CTest; run their binaries directly with +`--packages=/backends/native/test/op_samples`. The `Test Native Engines` CI workflow has separate Linux jobs for the shared runtime, CPU, GGML, and Vulkan, with each test suite run once. To reproduce with diff --git a/backends/native/runtime/deserialize/Samples.h b/backends/native/runtime/deserialize/Samples.h index bbbac2035c6..e2159616d09 100644 --- a/backends/native/runtime/deserialize/Samples.h +++ b/backends/native/runtime/deserialize/Samples.h @@ -23,6 +23,8 @@ struct SampleTensor { ScalarType dtype = kFloat; std::vector sizes; std::vector bytes; + + bool operator==(const SampleTensor&) const = default; }; struct Tolerance { diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index f6a72ffde51..719edc54578 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -65,6 +65,31 @@ add_custom_command( ) add_custom_target(native_op_samples DEPENDS ${_native_op_samples}.stamp) +if(EXECUTORCH_BUILD_NATIVE_GGML OR EXECUTORCH_BUILD_NATIVE_VULKAN) + if(NOT TARGET gflags AND NOT TARGET gflags::gflags) + find_package(gflags REQUIRED) + endif() + if(TARGET gflags::gflags) + set(_native_test_gflags gflags::gflags) + else() + set(_native_test_gflags gflags) + endif() + + # A gtest main judging the engine whose host a test links on the bundled + # samples of the packages it names; engine tests set its flags. + add_library(native_sample_test_main OBJECT samples/sample_test_main.cpp) + target_link_libraries( + native_sample_test_main + PUBLIC native_output_check native_runtime native_package native_engine + ${_native_test_gflags} GTest::gtest + ) + target_compile_options( + native_sample_test_main + PRIVATE $<$:/EHsc> + $<$>:-fexceptions> + ) +endif() + if(EXECUTORCH_BUILD_NATIVE_VULKAN) add_executable( native_vulkan_passes_test @@ -86,6 +111,23 @@ if(EXECUTORCH_BUILD_NATIVE_VULKAN) PRIVATE $<$:/EHsc> $<$>:-fexceptions> ) + + add_executable(native_vulkan_sample_test runtime/vulkan/vulkan_ptn_runner.cpp) + target_link_libraries( + native_vulkan_sample_test PRIVATE native_sample_test_main + native_vulkan_engine + ) + add_dependencies(native_vulkan_sample_test native_op_samples) + add_test(NAME native_vulkan_sample_test + COMMAND native_vulkan_sample_test --packages=${_native_op_samples} + ) + # TODO: Enable once every case passes. + set_tests_properties(native_vulkan_sample_test PROPERTIES DISABLED TRUE) + target_compile_options( + native_vulkan_sample_test + PRIVATE $<$:/EHsc> + $<$>:-fexceptions> + ) endif() if(EXECUTORCH_BUILD_NATIVE_GGML) @@ -101,6 +143,23 @@ if(EXECUTORCH_BUILD_NATIVE_GGML) PRIVATE $<$:/EHsc> $<$>:-fexceptions> ) + + add_executable(native_ggml_sample_test runtime/ggml/ggml_ptn_runner.cpp) + target_link_libraries( + native_ggml_sample_test PRIVATE native_sample_test_main native_ggml_engine + ) + add_dependencies(native_ggml_sample_test native_op_samples) + add_test(NAME native_ggml_sample_test + COMMAND native_ggml_sample_test --packages=${_native_op_samples} + --device=cpu + ) + # TODO: Enable once every case passes. + set_tests_properties(native_ggml_sample_test PROPERTIES DISABLED TRUE) + target_compile_options( + native_ggml_sample_test + PRIVATE $<$:/EHsc> + $<$>:-fexceptions> + ) endif() if(TARGET native_runner_utils) diff --git a/backends/native/test/runtime/ggml/BUCK b/backends/native/test/runtime/ggml/BUCK index 05de9fa2b72..71d998be09e 100644 --- a/backends/native/test/runtime/ggml/BUCK +++ b/backends/native/test/runtime/ggml/BUCK @@ -1,5 +1,6 @@ load("@fbcode_macros//build_defs:build_file_migration.bzl", "fbcode_target") load("@fbsource//third-party/llama.cpp:ggml_ci.bzl", "ggml_ci_labels") +load("@fbsource//tools/target_determinator/macros:ci.bzl", "ci") load("@fbsource//xplat/executorch/backends/native/runtime/engine:targets.bzl", "CPU_COMPAT", "CUDA_COMPAT", "HIP_COMPAT") load("@fbcode_macros//build_defs/lib:re_test_utils.bzl", "re_test_utils") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") @@ -23,7 +24,7 @@ fbcode_target( ) # The op suite runs once per ggml backend set; only the engine variant, the -# runner's --device, link deps and resources, the test host, and the CI +# runner's --device, link deps and test env, the test host, and the CI # configuration differ. _VARIANTS = [ {"suffix": "", "engine": "ggml_engine_cpu", "compat": CPU_COMPAT, "device": "cpu"}, @@ -40,10 +41,9 @@ _VARIANTS = [ # beside it, and its VMM pool calls the CUDA driver API. "deps": ["fbsource//third-party/llama.cpp:ggml_base[prefer-shared]"], "external_deps": [("cuda", None, "cuda_driver_api-lazy")], - # Buck does not put the CUDA driver stub's library on the runner's - # loader path, so it ships beside the runner. - "resources": { - "//tools/lazify:lazy_stub_binder[shared]": "libtools_lazify_lazy_stub_binder.so", + "env": { + # Buck does not put the CUDA driver stub's library on the loader path. + # @oss-disable[end= ]: "LD_PRELOAD": "$(location //tools/lazify:lazy_stub_binder[shared])", }, }, { @@ -91,16 +91,34 @@ _VARIANTS = [ for v in _VARIANTS ] +[ + fbcode_target( + _kind = runtime.cxx_test, + name = "ggml_sample_test" + v["suffix"], + args = [ + "--packages=$(location //executorch/backends/native/test/samples:op_samples)", + "--device=" + v["device"], + ], + env = v.get("env", {}), + remote_execution = v.get("remote_execution"), + # TODO: run in CI once they pass. + labels = ci.labels(ci.map(ci.skip_test())), + test_main = "//executorch/backends/native/test/samples:sample_test_main", + deps = [":ggml_engine_host" + v["suffix"]], + **v.get("compat", {}) + ) + for v in _VARIANTS +] + [ fbcode_target( _kind = runtime.python_test, name = "test_ggml_ops" + v["suffix"], srcs = ["test_ggml_ops.py"], - env = {"GGML_PTN_DEVICE": v["device"]}, + env = {"GGML_PTN_DEVICE": v["device"]} | v.get("env", {}), labels = v.get("labels", []), remote_execution = v.get("remote_execution"), - resources = {":ggml_ptn_runner" + v["suffix"]: "ggml_ptn_runner"} | - v.get("resources", {}), + resources = {":ggml_ptn_runner" + v["suffix"]: "ggml_ptn_runner"}, deps = [ "//caffe2:torch", "//executorch/exir/native:native", diff --git a/backends/native/test/runtime/ggml/test_ggml_ops.py b/backends/native/test/runtime/ggml/test_ggml_ops.py index 2a7169ffecd..1635f5e7082 100644 --- a/backends/native/test/runtime/ggml/test_ggml_ops.py +++ b/backends/native/test/runtime/ggml/test_ggml_ops.py @@ -334,18 +334,12 @@ def _run_ggml( command.append(f"--input_sizes={';'.join(map(_shape_text, steps))}") if graph_ops is not None: command.append("--print_graph_ops") - env = dict(os.environ) with as_file(files(__package__) / "ggml_ptn_runner") as runner: - # Shared libraries the runner needs may ship beside it. - env["LD_LIBRARY_PATH"] = ( - f"{Path(runner).parent}:{env.get('LD_LIBRARY_PATH', '')}" - ) result = subprocess.run( [str(runner), *command], capture_output=True, text=True, timeout=300, - env=env, ) if result.returncode != 0: raise AssertionError(f"ggml_ptn_runner failed:\n{result.stderr}") diff --git a/backends/native/test/runtime/vulkan/targets.bzl b/backends/native/test/runtime/vulkan/targets.bzl index 1e707ee89c0..33f56115527 100644 --- a/backends/native/test/runtime/vulkan/targets.bzl +++ b/backends/native/test/runtime/vulkan/targets.bzl @@ -1,3 +1,4 @@ +load("@fbsource//tools/target_determinator/macros:ci.bzl", "ci") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "is_xplat", "runtime") def define_common_targets(): @@ -60,6 +61,22 @@ def define_common_targets(): ], ) + # TODO: run in CI once it passes on SwiftShader. + runtime.cxx_test( + name = "vulkan_sample_test", + args = ["--packages=$(location //executorch/backends/native/test/samples:op_samples)"], + env = { + "ASAN_OPTIONS": "detect_leaks=0", + "LD_LIBRARY_PATH": "$(location :vulkan_test_driver)", + }, + labels = ci.labels(ci.map(ci.skip_test())), + test_main = "//executorch/backends/native/test/samples:sample_test_main", + deps = [ + ":vulkan_engine_host", + "fbsource//third-party/swiftshader/lib/linux-x64:libvk_swiftshader_fbcode", + ], + ) + runtime.cxx_test( name = "vulkan_engine_test", srcs = ["test_vulkan_engine.cpp"], diff --git a/backends/native/test/samples/BUCK b/backends/native/test/samples/BUCK index a59019b033c..122272f1d3e 100644 --- a/backends/native/test/samples/BUCK +++ b/backends/native/test/samples/BUCK @@ -66,3 +66,25 @@ fbcode_target( srcs = ["test_output_check.cpp"], deps = [":output_check"], ) + +# A gtest main judging the engine whose host library a test links on the +# bundled samples of the packages it names; engine tests set its flags. +fbcode_target( + _kind = runtime.cxx_library, + name = "sample_test_main", + srcs = ["sample_test_main.cpp"], + link_whole = True, + undefined_symbols = True, + visibility = ["//executorch/backends/native/test/..."], + exported_deps = ["//executorch/backends/native/test/runtime:engine_host"], + deps = [ + ":output_check", + "//executorch/backends/native/runtime:runtime", + "//executorch/backends/native/runtime/deserialize:package", + "//executorch/backends/native/runtime/deserialize:samples", + ], + external_deps = [ + "gflags", + "gtest", + ], +) diff --git a/backends/native/test/samples/sample_test_main.cpp b/backends/native/test/samples/sample_test_main.cpp new file mode 100644 index 00000000000..584676a49f6 --- /dev/null +++ b/backends/native/test/samples/sample_test_main.cpp @@ -0,0 +1,251 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// All rights reserved. +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +// A gtest main that runs the bundled samples of .ptn packages on the engine the +// binary links (see PtnRunner.h): one test per package and method, grouping +// samples by their first call. Each sample runs from a fresh context, and each +// method output is judged over all its samples by OutputCheck. +// +// Usage: +// --packages=a.ptn,dir [--rmse_ratio --max_ratio --rtol --sigmas] +// [engine host flags] [gtest flags] + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +DEFINE_string( + packages, + "", + "Comma-separated .ptn files or directories of them."); +DEFINE_double( + rmse_ratio, + ptn::testing::Policy{}.rmse_ratio, + "Allowed engine RMSE and biases, in multiples of eager's."); +DEFINE_double( + max_ratio, + ptn::testing::Policy{}.max_ratio, + "Allowed engine max abs error, in multiples of eager's."); +DEFINE_double( + rtol, + ptn::testing::Policy{}.rtol, + "Error allowed on top, relative to the precise outputs' magnitude."); +DEFINE_double( + sigmas, + ptn::testing::Policy{}.sigmas, + "Standard errors the fitted biases may deviate by."); + +namespace ptn::testing { +namespace { + +struct Bundle { + std::shared_ptr package; + std::shared_ptr program; + SampleSet golden; + SampleSet precise; +}; + +// Created on first use, so listing tests touches no device. +EngineHost& host() { + static const std::unique_ptr instance = [] { + std::unique_ptr made = make_engine_host(); + std::printf( + "engine %s on %s\n", made->name().c_str(), made->device_name().c_str()); + return made; + }(); + return *instance; +} + +// Keyed by method and output index. +using Checks = std::map, OutputCheck>; + +class SampleTest : public ::testing::Test { + public: + SampleTest(std::shared_ptr bundle, std::string method) + : bundle_(std::move(bundle)), method_(std::move(method)) {} + + void TestBody() override { + const std::vector& golden = bundle_->golden.samples; + const std::vector& precise = bundle_->precise.samples; + ASSERT_EQ(golden.size(), precise.size()); + Checks checks; + for (size_t s = 0; s < golden.size(); ++s) { + if (golden[s].calls.front().method != method_) { + continue; + } + SCOPED_TRACE("sample " + std::to_string(s)); + try { + run(golden[s], precise[s], checks); + } catch (const std::exception& e) { + FAIL() << e.what(); + } + if (HasFatalFailure()) { + return; + } + } + const Policy policy{ + .rmse_ratio = FLAGS_rmse_ratio, + .max_ratio = FLAGS_max_ratio, + .rtol = FLAGS_rtol, + .sigmas = FLAGS_sigmas}; + for (const auto& [key, check] : checks) { + const std::string name = key.first + " out " + std::to_string(key.second); + std::printf("%s: %s\n", name.c_str(), check.summary().c_str()); + for (const std::string& failure : check.failures(policy)) { + ADD_FAILURE() << name << ": " << failure; + } + } + } + + private: + void run(const Sample& golden, const Sample& precise, Checks& checks) { + ASSERT_EQ(golden.calls.size(), precise.calls.size()); + const std::unique_ptr context = + host().create_context(bundle_->program, bundle_->package); + std::map> executables; + for (size_t c = 0; c < golden.calls.size(); ++c) { + const SampleCall& call = golden.calls[c]; + ASSERT_EQ(call.method, precise.calls[c].method); + ASSERT_TRUE(call.inputs == precise.calls[c].inputs) + << call.method << " has golden and precise inputs that differ"; + ASSERT_TRUE(call.outputs && precise.calls[c].outputs) + << call.method << " has no recorded outputs"; + const std::vector& want = *call.outputs; + const std::vector& ref = *precise.calls[c].outputs; + std::unique_ptr& executable = executables[call.method]; + if (!executable) { + executable = context->compile(call.method); + } + ASSERT_EQ(executable->num_inputs(), call.inputs.size()); + ASSERT_EQ(executable->num_outputs(), want.size()); + ASSERT_EQ(ref.size(), want.size()); + + for (size_t i = 0; i < call.inputs.size(); ++i) { + const SampleTensor& in = call.inputs[i]; + ASSERT_STREQ( + scalar_type_name(executable->input_dtype(i)), + scalar_type_name(in.dtype)) + << "input " << i; + if (executable->input_sizes(i) != in.sizes) { + executable->resize_input(i, in.sizes); + } + executable->set_input( + i, + in.bytes.data(), + in.bytes.size() / element_size(in.dtype), + in.dtype); + } + executable->execute(); + for (size_t o = 0; o < want.size(); ++o) { + ASSERT_EQ(executable->output_sizes(o), want[o].sizes) << "output " << o; + ASSERT_EQ(ref[o].sizes, want[o].sizes) << "precise output " << o; + ASSERT_STREQ( + scalar_type_name(executable->output_dtype(o)), + scalar_type_name(want[o].dtype)) + << "output " << o; + std::vector got( + want[o].bytes.size() / element_size(want[o].dtype)); + executable->get_output(o, got.data(), got.size(), kFloat); + checks[{call.method, o}].add(got, want[o], ref[o]); + } + } + } + + std::shared_ptr bundle_; + std::string method_; +}; + +std::vector package_paths() { + std::vector paths; + std::stringstream list(FLAGS_packages); + for (std::string entry; std::getline(list, entry, ',');) { + if (entry.empty()) { + continue; + } + if (!std::filesystem::is_directory(entry)) { + paths.emplace_back(entry); + continue; + } + std::vector found; + for (const auto& file : std::filesystem::directory_iterator(entry)) { + if (file.path().extension() == ".ptn") { + found.push_back(file.path()); + } + } + std::ranges::sort(found); + paths.insert(paths.end(), found.begin(), found.end()); + } + return paths; +} + +void register_package(const std::filesystem::path& path) { + auto bundle = std::make_shared(); + bundle->package = + std::make_shared(Package::load(path.string())); + const ByteSpan program = bundle->package->program_bytes(); + bundle->program = std::make_shared( + Program::load(program.data(), program.size())); + const ZipReader zip = ZipReader::open(path.string()); + bundle->golden = read_sample_set(zip, "golden"); + bundle->precise = read_sample_set(zip, "precise"); + + std::set methods; + for (const Sample& sample : bundle->golden.samples) { + methods.insert(sample.calls.front().method); + } + const std::string suite = path.stem().string(); + for (const std::string& method : methods) { + ::testing::RegisterTest( + suite.c_str(), + method.c_str(), + nullptr, + nullptr, + __FILE__, + __LINE__, + [bundle, method]() { return new SampleTest(bundle, method); }); + } +} + +} // namespace +} // namespace ptn::testing + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + try { + const std::vector paths = + ptn::testing::package_paths(); + if (paths.empty()) { + throw std::invalid_argument("--packages names no .ptn"); + } + for (const std::filesystem::path& path : paths) { + ptn::testing::register_package(path); + } + } catch (const std::exception& e) { + std::fprintf(stderr, "error: %s\n", e.what()); + return 1; + } + return RUN_ALL_TESTS(); +} From cba25b180a6c0f063e33b144f7a826386862c27c Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:25 -0800 Subject: [PATCH 07/20] [executorch][native-ggml] Reject weight-only quantized linears unless activation quant is allowed Pull Request resolved: https://github.com/pytorch/executorch/pull/24361 ggml quantizes the activations of its quantized matmuls, so weight-only int4 and GGUF linears silently ran with numerics the PTN doesn't describe. Compile now rejects them unless `GgmlEngineOptions::allow_activation_quant_for_weight_only` is set. `ggml_ptn_runner` and `run_hf_llm_ggml` expose it as `--allow_activation_quant_for_weight_only`, which GGUF LLMs on ggml now need. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657742 @exported-using-ghexport Differential Revision: [D128254887](https://our.internmc.facebook.com/intern/diff/D128254887/) --- backends/native/README.md | 3 + .../hf_llm/runtime/run_hf_llm_ggml.cpp | 7 ++ .../native/runtime/engine/ggml/GgmlEngine.h | 5 ++ .../runtime/engine/ggml/GgmlExecutor.cpp | 20 ++++++ backends/native/test/e2e/test_cross_engine.py | 37 +++++++---- .../test/runtime/ggml/ggml_ptn_runner.cpp | 6 ++ .../test/runtime/ggml/test_ggml_engine.cpp | 64 ++++++++++++++----- .../native/test/runtime/ggml/test_ggml_ops.py | 38 +++++++++-- 8 files changed, 150 insertions(+), 30 deletions(-) diff --git a/backends/native/README.md b/backends/native/README.md index 8c51b12efac..8a2d76ce20a 100644 --- a/backends/native/README.md +++ b/backends/native/README.md @@ -339,6 +339,9 @@ install/bin/run_hf_llm_ggml --device cpu --cpu_threads 4 \ The GGML runner accepts `--device cpu|gpu`, `--device_index`, `--cpu_threads`, and `--max_cached_graphs` (default 1, 0 rebuilds each call). For GPU execution, enable the desired `GGML_*` backend when configuring ExecuTorch, as above. +GGML quantizes the activations of its quantized matmuls, so it rejects +weight-only quantized linears, including GGUF k-quants, unless +`--allow_activation_quant_for_weight_only` accepts that loss of precision. With `EXECUTORCH_BUILD_TESTS=ON`, `native_llm_runner_utils_test` exercises the LLM generation utilities. diff --git a/backends/native/examples/hf_llm/runtime/run_hf_llm_ggml.cpp b/backends/native/examples/hf_llm/runtime/run_hf_llm_ggml.cpp index b9c1c73233b..9115b9e0818 100644 --- a/backends/native/examples/hf_llm/runtime/run_hf_llm_ggml.cpp +++ b/backends/native/examples/hf_llm/runtime/run_hf_llm_ggml.cpp @@ -23,6 +23,11 @@ DEFINE_int32( max_cached_graphs, 1, "Graphs kept per executable by input shapes; 0 rebuilds every call."); +DEFINE_bool( + allow_activation_quant_for_weight_only, + false, + "Run weight-only quantized linears, such as GGUF k-quants, with ggml's " + "quantized activations."); int main(int argc, char** argv) { gflags::SetUsageMessage( @@ -41,6 +46,8 @@ int main(int argc, char** argv) { throw std::invalid_argument("--max_cached_graphs must be non-negative"); } options.max_cached_graphs = static_cast(FLAGS_max_cached_graphs); + options.allow_activation_quant_for_weight_only = + FLAGS_allow_activation_quant_for_weight_only; return ptn::GgmlEngineHost::create(options); }); } diff --git a/backends/native/runtime/engine/ggml/GgmlEngine.h b/backends/native/runtime/engine/ggml/GgmlEngine.h index 8fad5e69e17..61b64add99e 100644 --- a/backends/native/runtime/engine/ggml/GgmlEngine.h +++ b/backends/native/runtime/engine/ggml/GgmlEngine.h @@ -40,6 +40,11 @@ struct GgmlEngineOptions { // Prints each newly built graph's node count per ggml op to stderr, which // shows whether a lowering rewrite fired. bool print_graph_ops = false; + // ggml backends quantize the activations of a quantized matmul themselves, + // so a linear with quantized weights, int4 or GGUF, whose input is not a + // dequantized activation is rejected unless the caller accepts that loss of + // precision. + bool allow_activation_quant_for_weight_only = false; }; class GgmlEngineContext; diff --git a/backends/native/runtime/engine/ggml/GgmlExecutor.cpp b/backends/native/runtime/engine/ggml/GgmlExecutor.cpp index 66e5c2cf02a..b6537bd861a 100644 --- a/backends/native/runtime/engine/ggml/GgmlExecutor.cpp +++ b/backends/native/runtime/engine/ggml/GgmlExecutor.cpp @@ -542,7 +542,27 @@ class GgmlExecutable final : public EngineExecutable { "' may only feed linear or embedding weights, not '" + node.target + "'"); } + if (op == "aten.linear.default" && + !options_.allow_activation_quant_for_weight_only && + !has_dequantized_input(node)) { + throw std::runtime_error( + "ggml: weight-only quantized linear '" + node.name + + "' needs allow_activation_quant_for_weight_only, since ggml " + "quantizes its activations"); + } + } + } + + bool has_dequantized_input(const Node& linear) const { + const NamedArgument* input = find_arg(linear, "input", 0); + if (input == nullptr || input->arg.kind() != ArgKind::Tensor) { + return false; } + const NodeId producer = + method_.graph.value(input->arg.as_tensor().id).producer_id; + return valid(producer) && + op_key(method_.graph.node(producer).target) == + "torchao.dequantize_affine.default"; } // Converters read some dtypes through typed pointers, so the package's bytes diff --git a/backends/native/test/e2e/test_cross_engine.py b/backends/native/test/e2e/test_cross_engine.py index c6748064850..940f555a4bc 100644 --- a/backends/native/test/e2e/test_cross_engine.py +++ b/backends/native/test/e2e/test_cross_engine.py @@ -35,7 +35,14 @@ _MAX_SEQ = 16 -def _run(runner: str, package: Path, steps, outdir: Path, resize: bool = False): +def _run( + runner: str, + package: Path, + steps, + outdir: Path, + resize: bool = False, + flags: tuple[str, ...] = (), +): """Runs each step's inputs in order on one executable; returns each step's single float32 output.""" input_steps, output_steps, shapes = [], [], [] @@ -55,6 +62,7 @@ def _run(runner: str, package: Path, steps, outdir: Path, resize: bool = False): ] if resize: command.append(f"--input_sizes={';'.join(shapes)}") + command.extend(flags) resources = files(__package__) with as_file(resources / runner) as binary, as_file( resources / "vulkan_driver" @@ -390,11 +398,13 @@ def test_inplace_relu_of_dynamic_view(self): seq = Dim("seq", min=2, max=12) self._check_dynamic(_ViewRelu(), steps, ({1: seq},)) - def _check_engines(self, package, steps, expected, outdir, atol_fraction, engines): + def _check_engines( + self, package, steps, expected, outdir, atol_fraction, engines, flags=() + ): tokens = {} for runner in engines: with self.subTest(engine=runner): - outputs = _run(runner, package, steps, outdir, resize=True) + outputs = _run(runner, package, steps, outdir, resize=True, flags=flags) self.assertEqual(len(outputs), len(expected)) for step, (actual, reference) in enumerate(zip(outputs, expected)): self.assertEqual(actual.shape, reference.shape) @@ -433,6 +443,7 @@ def _check_llama( atol_fraction: float, engines=_ENGINES, max_seq: int = _MAX_SEQ, + flags: tuple[str, ...] = (), ): prompt = torch.randint(0, model.config.vocab_size, (1, 7)) eager = LastTokenLM(copy.deepcopy(model), max_seq, kv_cache_dtype).eval() @@ -457,7 +468,7 @@ def _check_llama( package = outdir / "tiny_llama.ptn" to_native(ep).save(str(package)) tokens = self._check_engines( - package, steps, expected, outdir, atol_fraction, engines + package, steps, expected, outdir, atol_fraction, engines, flags ) greedy = [e[0, -1].argmax().item() for e in expected] for runner, chosen in tokens.items(): @@ -472,16 +483,20 @@ def test_tiny_llama_gguf(self): model = _tiny_llama() pairs = _gguf_weights(model) load_gguf(model, "tiny.gguf", "llama", lambda _: iter(pairs)) - # ggml requantizes activations to Q8_K for its K-quant matmuls, while - # native-vk decodes the K-quant weights exactly to fp32. Re-seeding - # gives both engines the same prompt. - for runner, atol_fraction in ( - ("ggml_ptn_runner", 5e-2), - ("vulkan_ptn_runner", 1e-3), + # ggml requantizes activations to Q8_K for its K-quant matmuls, once + # allowed to, while native-vk decodes the K-quant weights exactly to + # fp32. Re-seeding gives both engines the same prompt. + for runner, atol_fraction, flags in ( + ("ggml_ptn_runner", 5e-2, ("--allow_activation_quant_for_weight_only",)), + ("vulkan_ptn_runner", 1e-3, ()), ): torch.manual_seed(0) self._check_llama( - copy.deepcopy(model), torch.float32, atol_fraction, engines=(runner,) + copy.deepcopy(model), + torch.float32, + atol_fraction, + engines=(runner,), + flags=flags, ) def test_tiny_llama_fp16_kv(self): diff --git a/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp b/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp index 85336504930..ad600382f04 100644 --- a/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp +++ b/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp @@ -26,6 +26,10 @@ DEFINE_bool( print_graph_ops, false, "Print each built graph's node count per ggml op to stderr."); +DEFINE_bool( + allow_activation_quant_for_weight_only, + false, + "Run weight-only quantized linears with ggml's quantized activations."); namespace ptn::testing { @@ -44,6 +48,8 @@ std::unique_ptr make_engine_host() { } options.max_cached_graphs = static_cast(FLAGS_max_cached_graphs); options.print_graph_ops = FLAGS_print_graph_ops; + options.allow_activation_quant_for_weight_only = + FLAGS_allow_activation_quant_for_weight_only; return GgmlEngineHost::create(options); } diff --git a/backends/native/test/runtime/ggml/test_ggml_engine.cpp b/backends/native/test/runtime/ggml/test_ggml_engine.cpp index a7498fc9a07..e78f6ec8cee 100644 --- a/backends/native/test/runtime/ggml/test_ggml_engine.cpp +++ b/backends/native/test/runtime/ggml/test_ggml_engine.cpp @@ -81,6 +81,11 @@ flatbuffers::Offset create_none_arg( constexpr size_t kQ4KBlockBytes = 144; +// The safetensors header of the affine linear program's constants. +constexpr const char* kQ4_0SafeTensorsHeader = + R"({"weight_scale":{"dtype":"F16","shape":[1,8],"data_offsets":[0,16]},)" + R"("weight":{"dtype":"U8","shape":[128],"data_offsets":[16,144]}})"; + // A [1, 256] x [256] linear with gguf:q4_k weights, or with symmetric int4 // AffineGroup weights whose fp16 scales `weight_scale` cover groups of 32. // `cache` is an unused mutable buffer. @@ -232,6 +237,10 @@ std::vector make_linear_program( builder.GetBufferPointer() + builder.GetSize()}; } +// For tests of quantized linears whose activation numerics aren't under test. +constexpr GgmlEngineOptions kAllowActivationQuant{ + .allow_activation_quant_for_weight_only = true}; + std::unique_ptr make_context( GgmlEngineHost& host, const std::vector& program_bytes) { @@ -473,7 +482,7 @@ std::vector make_cache_update_program( // cppcheck-suppress syntaxError TEST(GgmlEngineTest, ExecutesPackedQ4KLinear) { - auto host = GgmlEngineHost::create(); + auto host = GgmlEngineHost::create(kAllowActivationQuant); auto context = make_context(*host, make_linear_program()); auto executable = context->compile("forward"); @@ -491,8 +500,9 @@ TEST(GgmlEngineTest, ExecutesPackedQ4KLinear) { TEST(GgmlEngineTest, RepeatedExecutionsReadFreshInputs) { for (const size_t max_cached_graphs : {size_t{0}, size_t{1}}) { SCOPED_TRACE(max_cached_graphs); - auto host = GgmlEngineHost::create( - GgmlEngineOptions{.max_cached_graphs = max_cached_graphs}); + auto host = GgmlEngineHost::create(GgmlEngineOptions{ + .max_cached_graphs = max_cached_graphs, + .allow_activation_quant_for_weight_only = true}); auto context = make_context(*host, make_linear_program()); auto executable = context->compile("forward"); for (const float value : {1.0f, 2.0f, -1.0f}) { @@ -580,7 +590,7 @@ TEST(GgmlEngineTest, RejectsUnsupportedOpAtCompile) { } TEST(GgmlEngineTest, RejectsSecondExecutableBindingMutableBuffer) { - auto host = GgmlEngineHost::create(); + auto host = GgmlEngineHost::create(kAllowActivationQuant); auto context = make_context( *host, make_linear_program( @@ -590,12 +600,10 @@ TEST(GgmlEngineTest, RejectsSecondExecutableBindingMutableBuffer) { } void expect_compile_error( - const std::vector& program, + GgmlEngineContext& context, const std::string& message) { - auto host = GgmlEngineHost::create(); - auto context = make_context(*host, program); try { - context->compile("forward"); + context.compile("forward"); FAIL() << "compile accepted the program"; } catch (const std::runtime_error& error) { EXPECT_NE(std::string(error.what()).find(message), std::string::npos) @@ -603,6 +611,32 @@ void expect_compile_error( } } +void expect_compile_error( + const std::vector& program, + const std::string& message) { + auto host = GgmlEngineHost::create(); + expect_compile_error(*make_context(*host, program), message); +} + +TEST(GgmlEngineTest, RejectsWeightOnlyQuantizedLinearsByDefault) { + expect_compile_error( + make_linear_program(), "allow_activation_quant_for_weight_only"); + const std::vector program_bytes = make_linear_program( + "torch.ops.aten.linear.default", /*with_cache=*/false, /*affine=*/true); + auto host = GgmlEngineHost::create(); + auto context = host->create_ggml_context( + std::make_shared( + Program::load(program_bytes.data(), program_bytes.size())), + std::make_shared( + Package::load(OwnedBytes::from_vector(testing::make_zip({ + {kProgramEntry, program_bytes}, + {kSafeTensorsEntry, + testing::make_safetensors( + kQ4_0SafeTensorsHeader, /*scales + weights=*/16 + 128)}, + }))))); + expect_compile_error(*context, "allow_activation_quant_for_weight_only"); +} + TEST(GgmlEngineTest, RejectsIntegerGelu) { TestProgram program; program.input("x", fbs::ScalarType::LONG, {2, 3}); @@ -773,7 +807,7 @@ TEST(GgmlEngineTest, WriteToCachedMaskReachesLaterAttention) { } TEST(GgmlEngineTest, ReleasesMutableBufferWithItsExecutable) { - auto host = GgmlEngineHost::create(); + auto host = GgmlEngineHost::create(kAllowActivationQuant); auto context = make_context( *host, make_linear_program( @@ -1067,7 +1101,7 @@ TEST(GgmlEngineTest, Q4KLinearOfAHalfInput) { {{"input", std::string("x")}, {"weight", std::string("weight")}}, fbs::ScalarType::HALF, {1, 1}); - auto host = GgmlEngineHost::create(); + auto host = GgmlEngineHost::create(kAllowActivationQuant); auto context = make_context(*host, program.finish("y")); auto executable = context->compile("forward"); // The IEEE half bit pattern of 1.0. @@ -1399,7 +1433,9 @@ TEST(GgmlEngineTest, RejectsPackedWeightWithPartialBlocks) { {{"input", std::string("x")}, {"weight", std::string("weight")}}, fbs::ScalarType::FLOAT, {1, 1}); - expect_compile_error(program.finish("y"), "divisible by 256"); + auto host = GgmlEngineHost::create(kAllowActivationQuant); + expect_compile_error( + *make_context(*host, program.finish("y")), "divisible by 256"); } TEST(GgmlEngineTest, RejectsUnsupportedOpOnUnquantizedInput) { @@ -1478,9 +1514,7 @@ TEST(GgmlEngineTest, ViewsQ4_0ScalesOfMappedPackage) { // alternate fp16 scales 0.5 and 1.0. constexpr size_t kScaleBytes = 16; constexpr size_t kWeightBytes = 128; - std::string header = - R"({"weight_scale":{"dtype":"F16","shape":[1,8],"data_offsets":[0,16]},)" - R"("weight":{"dtype":"U8","shape":[128],"data_offsets":[16,144]}})"; + std::string header = kQ4_0SafeTensorsHeader; // The safetensors member comes first, so its payload starts right after // the local header, name, size prefix and JSON header. read_constant views // the scales in place only at an even offset. @@ -1522,7 +1556,7 @@ TEST(GgmlEngineTest, ViewsQ4_0ScalesOfMappedPackage) { ASSERT_TRUE(view.has_value()); ASSERT_EQ(reinterpret_cast(view->span().data()) % 2, misaligned); - auto host = GgmlEngineHost::create(); + auto host = GgmlEngineHost::create(kAllowActivationQuant); auto context = host->create_ggml_context( std::make_shared( Program::load(program_bytes.data(), program_bytes.size())), diff --git a/backends/native/test/runtime/ggml/test_ggml_ops.py b/backends/native/test/runtime/ggml/test_ggml_ops.py index 1635f5e7082..941a5de9268 100644 --- a/backends/native/test/runtime/ggml/test_ggml_ops.py +++ b/backends/native/test/runtime/ggml/test_ggml_ops.py @@ -305,6 +305,7 @@ def _run_ggml( max_cached_graphs: int = 1, graph_ops: collections.Counter | None = None, method: str = "forward", + allow_activation_quant_for_weight_only: bool = False, ): """Runs each step's inputs in order on one executable and returns each step's outputs as (bytes, shape) pairs. `graph_ops`, if given, collects @@ -334,6 +335,8 @@ def _run_ggml( command.append(f"--input_sizes={';'.join(map(_shape_text, steps))}") if graph_ops is not None: command.append("--print_graph_ops") + if allow_activation_quant_for_weight_only: + command.append("--allow_activation_quant_for_weight_only") with as_file(files(__package__) / "ggml_ptn_runner") as runner: result = subprocess.run( [str(runner), *command], @@ -387,15 +390,35 @@ def test_gguf_q6_k_linear(self): # block scale on GPUs. Test the weight layout tightly with exact # activations, then separately bound the randomized requantization loss. gpu = os.environ.get("GGML_PTN_DEVICE", "cpu") == "gpu" - self._check(model, (representable,), atol=5e-4, rtol=2e-4 if gpu else 1e-5) - self._check(model, (torch.randn(2, 256) / 8,), atol=0.35, rtol=0.02) + self._check( + model, + (representable,), + atol=5e-4, + rtol=2e-4 if gpu else 1e-5, + allow_activation_quant_for_weight_only=True, + ) + self._check( + model, + (torch.randn(2, 256) / 8,), + atol=0.35, + rtol=0.02, + allow_activation_quant_for_weight_only=True, + ) def test_gguf_q6_k_embedding(self): model = torch.nn.Embedding(8, 256) model.weight = torch.nn.Parameter(self._q6_k_weights(8), requires_grad=False) self._check(model, (torch.tensor([[0, 3, 6], [7, 1, 4]]),)) - def _check(self, model, inputs, atol=1e-5, rtol=1e-5, graph_ops=None): + def _check( + self, + model, + inputs, + atol=1e-5, + rtol=1e-5, + graph_ops=None, + allow_activation_quant_for_weight_only=False, + ): model = model.eval() # Eager writes buffers in place; the package must keep them as constructed. with torch.no_grad(): @@ -406,7 +429,14 @@ def _check(self, model, inputs, atol=1e-5, rtol=1e-5, graph_ops=None): package = outdir / "model.ptn" to_native(torch.export.export(model, inputs)).save(str(package)) (outputs,) = _run_ggml( - package, [inputs], outdir, len(expected), graph_ops=graph_ops + package, + [inputs], + outdir, + len(expected), + graph_ops=graph_ops, + allow_activation_quant_for_weight_only=( + allow_activation_quant_for_weight_only + ), ) self.assertEqual(len(outputs), len(expected)) for (raw, shape), reference in zip(outputs, expected): From 7526cf8c6bfde338e72de62e2767458f1cf6d39e Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:26 -0800 Subject: [PATCH 08/20] [executorch][native] Judge activation-quantized engines against an int8-activation golden Pull Request resolved: https://github.com/pytorch/executorch/pull/24362 With activation quantization allowed, ggml's weight-only linears no longer compute what `golden` describes. Weight-only packages now carry `golden-int8_act`, eager with int8 per-token dynamic activations, which `sample_test_main` judges against under `--allow_activation_quant_for_weight_only`. The flag is a `HostOptions` field that Vulkan rejects; new `ggml_sample_test_act_quant` targets and the `native_ggml_sample_test_act_quant` ctest set it. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657755 @exported-using-ghexport Differential Revision: [D128254888](https://our.internmc.facebook.com/intern/diff/D128254888/) --- backends/native/test/CMakeLists.txt | 6 +++++ backends/native/test/runtime/PtnRunner.h | 9 ++++++- backends/native/test/runtime/ggml/BUCK | 10 +++++-- .../test/runtime/ggml/ggml_ptn_runner.cpp | 8 ++---- .../native/test/runtime/ptn_runner_main.cpp | 8 +++++- .../test/runtime/vulkan/vulkan_ptn_runner.cpp | 9 ++++++- backends/native/test/samples/record.py | 23 +++++++++++++--- .../native/test/samples/sample_test_main.cpp | 26 +++++++++++++++---- backends/native/test/samples/schemes.py | 17 ++++++++++++ backends/native/test/samples/test_record.py | 19 ++++++++++++++ 10 files changed, 115 insertions(+), 20 deletions(-) diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index 719edc54578..83e144150aa 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -155,6 +155,12 @@ if(EXECUTORCH_BUILD_NATIVE_GGML) ) # TODO: Enable once every case passes. set_tests_properties(native_ggml_sample_test PROPERTIES DISABLED TRUE) + # native_ggml_sample_test_act_quant lets ggml quantize the activations of + # weight-only quantized linears, judged against golden-int8_act. + add_test(NAME native_ggml_sample_test_act_quant + COMMAND native_ggml_sample_test --packages=${_native_op_samples} + --device=cpu --allow_activation_quant_for_weight_only + ) target_compile_options( native_ggml_sample_test PRIVATE $<$:/EHsc> diff --git a/backends/native/test/runtime/PtnRunner.h b/backends/native/test/runtime/PtnRunner.h index de23aeb4da4..7bb03987f7a 100644 --- a/backends/native/test/runtime/PtnRunner.h +++ b/backends/native/test/runtime/PtnRunner.h @@ -12,8 +12,15 @@ namespace ptn::testing { +// Numerics a driver lets an engine trade away. Drivers then judge the engine as +// having made the trade, so an engine that never makes it rejects the option. +struct HostOptions { + // Quantize the activations of linears whose weights alone are quantized. + bool allow_activation_quant_for_weight_only = false; +}; + // Defined once per binary by an engine's host library; drivers such as // ptn_runner_main.cpp run every engine through the host it returns. -std::unique_ptr make_engine_host(); +std::unique_ptr make_engine_host(const HostOptions& options); } // namespace ptn::testing diff --git a/backends/native/test/runtime/ggml/BUCK b/backends/native/test/runtime/ggml/BUCK index 71d998be09e..a740ae2fcf1 100644 --- a/backends/native/test/runtime/ggml/BUCK +++ b/backends/native/test/runtime/ggml/BUCK @@ -91,14 +91,16 @@ _VARIANTS = [ for v in _VARIANTS ] +# ggml_sample_test_act_quant lets ggml quantize the activations of weight-only +# quantized linears, judged against golden-int8_act. [ fbcode_target( _kind = runtime.cxx_test, - name = "ggml_sample_test" + v["suffix"], + name = "ggml_sample_test" + name + v["suffix"], args = [ "--packages=$(location //executorch/backends/native/test/samples:op_samples)", "--device=" + v["device"], - ], + ] + flags, env = v.get("env", {}), remote_execution = v.get("remote_execution"), # TODO: run in CI once they pass. @@ -108,6 +110,10 @@ _VARIANTS = [ **v.get("compat", {}) ) for v in _VARIANTS + for name, flags in [ + ("", []), + ("_act_quant", ["--allow_activation_quant_for_weight_only"]), + ] ] [ diff --git a/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp b/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp index ad600382f04..9bff34d2f66 100644 --- a/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp +++ b/backends/native/test/runtime/ggml/ggml_ptn_runner.cpp @@ -26,15 +26,11 @@ DEFINE_bool( print_graph_ops, false, "Print each built graph's node count per ggml op to stderr."); -DEFINE_bool( - allow_activation_quant_for_weight_only, - false, - "Run weight-only quantized linears with ggml's quantized activations."); namespace ptn::testing { // cppcheck-suppress unusedFunction -std::unique_ptr make_engine_host() { +std::unique_ptr make_engine_host(const HostOptions& host_options) { GgmlEngineOptions options; if (FLAGS_device == "gpu") { options.device = GgmlDevice::Gpu; @@ -49,7 +45,7 @@ std::unique_ptr make_engine_host() { options.max_cached_graphs = static_cast(FLAGS_max_cached_graphs); options.print_graph_ops = FLAGS_print_graph_ops; options.allow_activation_quant_for_weight_only = - FLAGS_allow_activation_quant_for_weight_only; + host_options.allow_activation_quant_for_weight_only; return GgmlEngineHost::create(options); } diff --git a/backends/native/test/runtime/ptn_runner_main.cpp b/backends/native/test/runtime/ptn_runner_main.cpp index 654ef0ad569..973aeec5025 100644 --- a/backends/native/test/runtime/ptn_runner_main.cpp +++ b/backends/native/test/runtime/ptn_runner_main.cpp @@ -65,6 +65,10 @@ DEFINE_int32( iterations, 1, "Timed executions; above one, prints median and min latency."); +DEFINE_bool( + allow_activation_quant_for_weight_only, + false, + "Let the engine quantize the activations of weight-only quantized linears."); namespace { @@ -201,7 +205,9 @@ void run() { const ptn::ByteSpan program_bytes = package->program_bytes(); const auto program = std::make_shared( ptn::Program::load(program_bytes.data(), program_bytes.size())); - const auto host = ptn::testing::make_engine_host(); + const auto host = ptn::testing::make_engine_host( + {.allow_activation_quant_for_weight_only = + FLAGS_allow_activation_quant_for_weight_only}); const auto context = host->create_context(program, package); const auto executable = context->compile(FLAGS_method); diff --git a/backends/native/test/runtime/vulkan/vulkan_ptn_runner.cpp b/backends/native/test/runtime/vulkan/vulkan_ptn_runner.cpp index 3f8b9e76ee0..51163c0d478 100644 --- a/backends/native/test/runtime/vulkan/vulkan_ptn_runner.cpp +++ b/backends/native/test/runtime/vulkan/vulkan_ptn_runner.cpp @@ -6,13 +6,20 @@ // The Vulkan engine host for PTN test drivers. +#include + #include #include namespace ptn::testing { // cppcheck-suppress unusedFunction -std::unique_ptr make_engine_host() { +std::unique_ptr make_engine_host(const HostOptions& options) { + if (options.allow_activation_quant_for_weight_only) { + throw std::invalid_argument( + "vulkan: never quantizes activations, so it rejects " + "allow_activation_quant_for_weight_only"); + } return VulkanEngineHost::create(); } diff --git a/backends/native/test/samples/record.py b/backends/native/test/samples/record.py index 6846890f081..d3565a01667 100644 --- a/backends/native/test/samples/record.py +++ b/backends/native/test/samples/record.py @@ -9,9 +9,10 @@ A package holds one method per case and dtype under one quantization scheme, and two sample sets over the same inputs: `golden`, the eager outputs, and `precise`, the same math in float64 with weights dequantized and activations -unquantized, floating outputs stored as float32. Nothing in it depends on an -engine or a device: consumers judge an engine by its distance from `precise` -relative to eager's. +unquantized, floating outputs stored as float32. Weight-only schemes add +`golden-int8_act`, the eager outputs with the same weights and int8 per-token +dynamic activations. Nothing in it depends on an engine or a device: consumers +judge an engine by its distance from `precise` relative to eager's. """ import copy @@ -50,7 +51,8 @@ def record( dtypes: Iterable[str] = tuple(DTYPES), ) -> None: """Writes no package, removing any at `path`, when the scheme fits no case.""" - programs, golden, precise = {}, [], [] + act_scheme = scheme.act_quant() + programs, golden, precise, golden_act = {}, [], [], [] for case in cases: if not scheme.fits(case.model): logger.info(f"{path.name}: {case.name} does not fit the scheme") @@ -63,6 +65,9 @@ def record( model = copy.deepcopy(case.model).to(dtype).eval() scheme.quantize(model) method = _Method({name: model}) + if act_scheme is not None: + act_model = copy.deepcopy(case.model).to(dtype).eval() + act_scheme.quantize(act_model) state = { k: scheme.decode(v).double() for k, v in method.state_dict().items() } @@ -76,6 +81,10 @@ def record( t.float() if t.is_floating_point() else t for t in outputs(ref) ] precise.append(Sample([Call(name, list(x), ref)])) + if act_scheme is not None: + golden_act.append( + Sample([Call(name, list(x), outputs(act_model(*x)))]) + ) if not programs: logger.info(f"{path.name}: no case fits the scheme, not written") path.unlink(missing_ok=True) @@ -90,3 +99,9 @@ def record( precise, "float64, weights dequantized, activations unquantized", source ), ) + if act_scheme is not None: + write_samples( + str(path), + "golden-int8_act", + SampleSet(golden_act, "eager, int8 per-token dynamic activations", source), + ) diff --git a/backends/native/test/samples/sample_test_main.cpp b/backends/native/test/samples/sample_test_main.cpp index 584676a49f6..e44d987343c 100644 --- a/backends/native/test/samples/sample_test_main.cpp +++ b/backends/native/test/samples/sample_test_main.cpp @@ -11,7 +11,8 @@ // // Usage: // --packages=a.ptn,dir [--rmse_ratio --max_ratio --rtol --sigmas] -// [engine host flags] [gtest flags] +// [--allow_activation_quant_for_weight_only] [engine host flags] +// [gtest flags] #include #include @@ -56,6 +57,11 @@ DEFINE_double( sigmas, ptn::testing::Policy{}.sigmas, "Standard errors the fitted biases may deviate by."); +DEFINE_bool( + allow_activation_quant_for_weight_only, + false, + "Let the engine quantize the activations of weight-only quantized linears, " + "judging it against golden-int8_act samples where a package has them."); namespace ptn::testing { namespace { @@ -63,6 +69,7 @@ namespace { struct Bundle { std::shared_ptr package; std::shared_ptr program; + std::string golden_set; SampleSet golden; SampleSet precise; }; @@ -70,7 +77,9 @@ struct Bundle { // Created on first use, so listing tests touches no device. EngineHost& host() { static const std::unique_ptr instance = [] { - std::unique_ptr made = make_engine_host(); + std::unique_ptr made = make_engine_host( + {.allow_activation_quant_for_weight_only = + FLAGS_allow_activation_quant_for_weight_only}); std::printf( "engine %s on %s\n", made->name().c_str(), made->device_name().c_str()); return made; @@ -111,7 +120,8 @@ class SampleTest : public ::testing::Test { .rtol = FLAGS_rtol, .sigmas = FLAGS_sigmas}; for (const auto& [key, check] : checks) { - const std::string name = key.first + " out " + std::to_string(key.second); + const std::string name = key.first + " out " + + std::to_string(key.second) + " vs " + bundle_->golden_set; std::printf("%s: %s\n", name.c_str(), check.summary().c_str()); for (const std::string& failure : check.failures(policy)) { ADD_FAILURE() << name << ": " << failure; @@ -129,7 +139,8 @@ class SampleTest : public ::testing::Test { const SampleCall& call = golden.calls[c]; ASSERT_EQ(call.method, precise.calls[c].method); ASSERT_TRUE(call.inputs == precise.calls[c].inputs) - << call.method << " has golden and precise inputs that differ"; + << call.method << " has " << bundle_->golden_set + << " and precise inputs that differ"; ASSERT_TRUE(call.outputs && precise.calls[c].outputs) << call.method << " has no recorded outputs"; const std::vector& want = *call.outputs; @@ -208,7 +219,12 @@ void register_package(const std::filesystem::path& path) { bundle->program = std::make_shared( Program::load(program.data(), program.size())); const ZipReader zip = ZipReader::open(path.string()); - bundle->golden = read_sample_set(zip, "golden"); + // Eager outputs with int8 per-token dynamic activations, which packages with + // weight-only quantized linears also carry. + const bool act_quant_golden = FLAGS_allow_activation_quant_for_weight_only && + zip.member_size("samples/golden-int8_act/index.json").has_value(); + bundle->golden_set = act_quant_golden ? "golden-int8_act" : "golden"; + bundle->golden = read_sample_set(zip, bundle->golden_set); bundle->precise = read_sample_set(zip, "precise"); std::set methods; diff --git a/backends/native/test/samples/schemes.py b/backends/native/test/samples/schemes.py index 8966e7facad..82d9c2d8cec 100644 --- a/backends/native/test/samples/schemes.py +++ b/backends/native/test/samples/schemes.py @@ -33,6 +33,11 @@ def decode(self, value: torch.Tensor) -> torch.Tensor: """A state tensor of a quantized model as a plain tensor.""" return value + def act_quant(self) -> "Scheme | None": + """This weight-only scheme's weights with int8 dynamic activations, or + None when the scheme is not weight-only.""" + return None + def _fits(model: torch.nn.Module, group: int | None) -> bool: """Whether `model` has an nn.Linear and every one's in_features split into @@ -68,6 +73,18 @@ def quantize(self, model: torch.nn.Module) -> None: def decode(self, value: torch.Tensor) -> torch.Tensor: return value.dequantize() if isinstance(value, TorchAOBaseTensor) else value + def act_quant(self) -> Scheme | None: + if not isinstance(self.config, IntxWeightOnlyConfig): + return None + return Torchao( + Int8DynamicActivationIntxWeightConfig( + weight_dtype=self.config.weight_dtype, + weight_granularity=self.config.granularity, + weight_mapping_type=self.config.mapping_type, + weight_scale_dtype=self.config.scale_dtype, + ) + ) + def schemes() -> dict[str, Scheme]: return { diff --git a/backends/native/test/samples/test_record.py b/backends/native/test/samples/test_record.py index 391ecd9b52a..086f2a55d9b 100644 --- a/backends/native/test/samples/test_record.py +++ b/backends/native/test/samples/test_record.py @@ -77,6 +77,25 @@ def test_record_golden_is_eager_and_precise_is_float64(self) -> None: atol=0, ) + def test_record_adds_the_8da4w_golden_to_weight_only_packages(self) -> None: + case = Case("linear", torch.nn.Linear(64, 8), draws(2, (3, 64))) + sets = {} + with tempfile.TemporaryDirectory() as tmp: + for name in ("float", "int4wo_g32", "8da4w_g32"): + path = Path(tmp) / f"{name}.ptn" + record([case], schemes()[name], path, dtypes=["f32"]) + sets[name] = read_samples(str(path)) + self.assertNotIn("golden-int8_act", sets["float"]) + self.assertNotIn("golden-int8_act", sets["8da4w_g32"]) + for got, want in zip( + sets["int4wo_g32"]["golden-int8_act"].samples, + sets["8da4w_g32"]["golden"].samples, + strict=True, + ): + torch.testing.assert_close( + got.calls[0].outputs, want.calls[0].outputs, rtol=0, atol=0 + ) + def test_record_keeps_integer_outputs_precise(self) -> None: class Argmax(torch.nn.Module): def forward(self, x: torch.Tensor) -> torch.Tensor: From 29b275f277f69e35056152a17381837c24645517 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:26 -0800 Subject: [PATCH 09/20] [executorch][native-ggml] Compute cuBLAS GEMMs in f32 in the CUDA sample tests Pull Request resolved: https://github.com/pytorch/executorch/pull/24363 By default ggml-CUDA's cuBLAS GEMMs compute in f16 for f16 and dequantized weights, and in TF32 for f32 ones, so the linear samples measured those defaults rather than ggml's kernels. The CUDA sample-test targets set `GGML_CUDA_CUBLAS_COMPUTE_TYPE=f32` and `NVIDIA_TF32_OVERRIDE=0`; llama.cpp's defaults, `test_ggml_ops` and the bundle are unchanged. Authored with Claude Code. Differential Revision: [D128254889](https://our.internmc.facebook.com/intern/diff/D128254889/) **NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D128254889/)! ghstack-source-id: 463657744 --- backends/native/test/runtime/ggml/BUCK | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backends/native/test/runtime/ggml/BUCK b/backends/native/test/runtime/ggml/BUCK index a740ae2fcf1..ad23a997116 100644 --- a/backends/native/test/runtime/ggml/BUCK +++ b/backends/native/test/runtime/ggml/BUCK @@ -45,6 +45,11 @@ _VARIANTS = [ # Buck does not put the CUDA driver stub's library on the loader path. # @oss-disable[end= ]: "LD_PRELOAD": "$(location //tools/lazify:lazy_stub_binder[shared])", }, + # ggml-CUDA computes f16 BLAS GEMMs in f16 and f32 ones in TF32 by default. + "sample_env": { + "GGML_CUDA_CUBLAS_COMPUTE_TYPE": "f32", + "NVIDIA_TF32_OVERRIDE": "0", + }, }, { "suffix": "_hip", @@ -101,7 +106,7 @@ _VARIANTS = [ "--packages=$(location //executorch/backends/native/test/samples:op_samples)", "--device=" + v["device"], ] + flags, - env = v.get("env", {}), + env = v.get("env", {}) | v.get("sample_env", {}), remote_execution = v.get("remote_execution"), # TODO: run in CI once they pass. labels = ci.labels(ci.map(ci.skip_test())), From b548c687f1724d67247921f700f4dbe7db97b7b4 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:27 -0800 Subject: [PATCH 10/20] [executorch][native] Suggest rounding to nearest when engines narrow values Pull Request resolved: https://github.com/pytorch/executorch/pull/24364 Rounding toward zero when narrowing f32 to f16, which Vulkan and WebGPU allow, biases every stored value toward zero, and the bias compounds across layers. The README suggests rounding to nearest, ties to even, as eager does. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657754 @exported-using-ghexport Differential Revision: [D128254885](https://our.internmc.facebook.com/intern/diff/D128254885/) --- backends/native/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backends/native/README.md b/backends/native/README.md index 8a2d76ce20a..d3c9e87f283 100644 --- a/backends/native/README.md +++ b/backends/native/README.md @@ -145,6 +145,14 @@ an engine through the interfaces in `runtime/engine/Engine.h`: An engine rejects a method whose semantics it cannot fully honor. +Engines need not match eager bit for bit: accumulation precision and order, +fused multiply-add, and the accuracy of functions such as `exp` are up to each +engine. As a guideline, an engine should round to nearest when it narrows a +value for storage, with ties to even between floating-point types (f32 to f16 or +bf16), as eager does. Rounding toward zero, which some GPU APIs allow, biases +every stored value toward zero, and the bias compounds across layers instead of +averaging out. + | Engine | Source | Notes | | --- | --- | --- | | Vulkan | `runtime/vulkan/` | Builds on the ET-VK `ComputeGraph` and shaders. | From 1ff81304b19acce659db17cda7835441f8999cff Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:28 -0800 Subject: [PATCH 11/20] [executorch][native] Add FACTO-drawn linear cases to the op samples Pull Request resolved: https://github.com/pytorch/executorch/pull/24365 The hand-written cases cover few shapes. `facto_cases` turns the argument tuples FACTO draws for an op spec from a fixed seed into one-sample cases, `facto_`, and `linear.py` keeps every fifth `linear.default` tuple. The spec lives here, so ET backend tests are unchanged. The `native_op_samples` CMake target puts the vendored FACTO submodule on `PYTHONPATH` and reruns when its sources change. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657759 @exported-using-ghexport Differential Revision: [D128254881](https://our.internmc.facebook.com/intern/diff/D128254881/) --- backends/native/README.md | 5 +- backends/native/test/CMakeLists.txt | 8 ++- backends/native/test/samples/BUCK | 1 + backends/native/test/samples/cases.py | 22 ++++++++ backends/native/test/samples/linear.py | 77 ++++++++++++++++++++++++-- 5 files changed, 104 insertions(+), 9 deletions(-) diff --git a/backends/native/README.md b/backends/native/README.md index d3c9e87f283..c918183cddc 100644 --- a/backends/native/README.md +++ b/backends/native/README.md @@ -217,8 +217,9 @@ enabled, `native_runner_utils_test` covers their shared runner utilities. Run th with `ctest --test-dir cmake-out -R native_` after `cmake --build cmake-out`. The engines' sample tests judge them on the bundled op samples. Building them runs -the sample generator, which needs `torch` and `torchao`. Those with failing cases -are disabled in CTest; run their binaries directly with +the sample generator, which needs `torch`, `torchao` and the FACTO submodule at +`backends/cadence/utils/FACTO`. Those with failing cases are disabled in CTest; +run their binaries directly with `--packages=/backends/native/test/op_samples`. The `Test Native Engines` CI workflow has separate Linux jobs for the shared diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index 83e144150aa..523d450ba50 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -48,16 +48,18 @@ endforeach() # environment pins the eager kernels' ISA and threads, so the golden is the same # on any x86 host with AVX2. Other hosts can differ. set(_native_op_samples ${CMAKE_CURRENT_BINARY_DIR}/op_samples) -file(GLOB _native_op_samples_srcs CONFIGURE_DEPENDS +file(GLOB_RECURSE _native_op_samples_srcs CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/samples/*.py + ${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO/facto/*.py ) add_custom_command( OUTPUT ${_native_op_samples}.stamp COMMAND ${CMAKE_COMMAND} -E env MKL_CBWR=COMPATIBLE ONEDNN_MAX_CPU_ISA=AVX2 ATEN_CPU_CAPABILITY=avx2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 - "PYTHONPATH=${EXECUTORCH_ROOT}/src" ${PYTHON_EXECUTABLE} -m - executorch.backends.native.test.samples.generate --out ${_native_op_samples} + "PYTHONPATH=${EXECUTORCH_ROOT}/src:${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO" + ${PYTHON_EXECUTABLE} -m executorch.backends.native.test.samples.generate + --out ${_native_op_samples} COMMAND ${CMAKE_COMMAND} -E touch ${_native_op_samples}.stamp DEPENDS ${_native_op_samples_srcs} ${EXECUTORCH_ROOT}/backends/test/suite/operators/test_linear.py diff --git a/backends/native/test/samples/BUCK b/backends/native/test/samples/BUCK index 122272f1d3e..326d33d26de 100644 --- a/backends/native/test/samples/BUCK +++ b/backends/native/test/samples/BUCK @@ -18,6 +18,7 @@ fbcode_target( "//executorch/backends/test/suite/operators:test_operators", "//executorch/exir/native:native", "//pytorch/ao:torchao", + "//pytorch/facto:facto", ], ) diff --git a/backends/native/test/samples/cases.py b/backends/native/test/samples/cases.py index b08baba963a..82b97316b24 100644 --- a/backends/native/test/samples/cases.py +++ b/backends/native/test/samples/cases.py @@ -7,11 +7,16 @@ """Op cases: a model and the inputs of each sample it is recorded with.""" import inspect +from collections.abc import Callable from dataclasses import dataclass from types import ModuleType +from typing import TYPE_CHECKING import torch +if TYPE_CHECKING: + from facto.inputgen.specs.model import Spec + Inputs = tuple[torch.Tensor, ...] # Caps a suite case's input and output elements over its samples, keeping at @@ -85,3 +90,20 @@ def suite_cases(module: ModuleType, samples: int) -> list[Case]: if not cases: raise ValueError(f"no test in {module.__name__} lowers a model") return cases + + +def facto_cases( + spec: "Spec", build: Callable[..., Case], keep: Callable[[int, tuple], bool] +) -> list[Case]: + """One-sample cases `build(name, *args)` from FACTO's argument tuples for + `spec`, named by position, keeping those where `keep(position, args)`.""" + # FACTO is on the path for the sample generator only, not for pytest. + from facto.inputgen.argtuple.gen import ArgumentTupleGenerator + from facto.inputgen.utils.random_manager import seeded_random_manager + + seeded_random_manager.seed(0) + return [ + build(f"facto_{i}", *args) + for i, (args, _, _) in enumerate(ArgumentTupleGenerator(spec).gen()) + if keep(i, args) + ] diff --git a/backends/native/test/samples/linear.py b/backends/native/test/samples/linear.py index 24c39fe2ccc..c1f1dc60e48 100644 --- a/backends/native/test/samples/linear.py +++ b/backends/native/test/samples/linear.py @@ -4,12 +4,76 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -"""Linear cases: the backend test suite's, and shapes and inputs that stress -accumulation and quantization.""" +"""Linear cases: the backend test suite's, FACTO's, and shapes and inputs that +stress accumulation and quantization.""" import torch -from executorch.backends.native.test.samples.cases import Case, draws, suite_cases +from executorch.backends.native.test.samples.cases import ( + Case, + draws, + facto_cases, + suite_cases, +) from executorch.backends.test.suite.operators import test_linear +from facto.inputgen.argument.type import ArgType +from facto.inputgen.specs.model import ConstraintProducer as cp, InPosArg, OutArg, Spec + + +def _unit(*constraints) -> list: + """Float32 values in [-1, 1] under `constraints`.""" + return [ + cp.Dtype.Eq(lambda deps: torch.float32), + *constraints, + cp.Value.Ge(lambda deps, dtype, struct: -1), + cp.Value.Le(lambda deps, dtype, struct: 1), + ] + + +# in_features is a multiple of every scheme's group size, and out_features is +# large enough to fit an error's abs_bias and rel_bias from one sample. +_FACTO_SPEC = Spec( + op="linear.default", + inspec=[ + InPosArg( + ArgType.Tensor, + name="input", + deps=[1], + constraints=_unit( + cp.Rank.Ge(lambda deps: 1), + cp.Rank.Le(lambda deps: 3), + cp.Size.In( + lambda deps, r, d: [deps[0].size(1)] if d == r - 1 else range(1, 9) + ), + ), + ), + InPosArg( + ArgType.Tensor, + name="weight", + constraints=_unit( + cp.Rank.Eq(lambda deps: 2), + cp.Size.In(lambda deps, r, d: [128, 256, 384] if d else [16, 48, 96]), + ), + ), + InPosArg( + ArgType.TensorOpt, + name="bias", + deps=[1], + constraints=_unit( + cp.Rank.Eq(lambda deps: 1), + cp.Size.Eq(lambda deps, r, d: deps[0].size(0)), + ), + ), + ], + outspec=[OutArg(ArgType.Tensor)], +) + + +def _facto_case(name: str, x, weight, bias) -> Case: + model = torch.nn.Linear(weight.size(1), weight.size(0), bias=bias is not None) + model.weight.data = weight + if bias is not None: + model.bias.data = bias + return Case(name, model, [(x,)]) def cases() -> list[Case]: @@ -22,7 +86,8 @@ def cases() -> list[Case]: zero_row = draws(2, (8, 256)) for (x,) in zero_row: x[2] = 0 - return suite + [ + return [ + *suite, Case("decode", torch.nn.Linear(2048, 512), draws(8, (1, 2048))), Case("prefill", torch.nn.Linear(1024, 256), draws(2, (64, 1024))), Case("large_k", torch.nn.Linear(8192, 32), draws(4, (4, 8192))), @@ -33,4 +98,8 @@ def cases() -> list[Case]: ), Case("outlier_channel", torch.nn.Linear(256, 128), outlier), Case("zero_row", torch.nn.Linear(256, 64), zero_row), + # FACTO varies one argument at a time, so only every fifth tuple is kept. + # Its values lie on a 1/8 grid, which makes float32 math exact: these + # cases cover shapes more than rounding. + *facto_cases(_FACTO_SPEC, _facto_case, lambda i, _: i % 5 == 0), ] From 7202d58955807ceedd259cc7d238f6701fc37353 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:28 -0800 Subject: [PATCH 12/20] [executorch][native] Add GGUF q4_k and q6_k packages to the op samples Pull Request resolved: https://github.com/pytorch/executorch/pull/24366 PTN carries GGUF k-quant constants, but no op package exercised them. A `Gguf` scheme, as `gguf_q4_k` and `gguf_q6_k`, writes random valid blocks to a .gguf with gguf-py and loads them back as a checkpoint would. `golden-int8_act` now comes from per-token int8 pre-hooks on the weight-only model, which also work for GGUF, so `Scheme.weight_only` replaces `Scheme.act_quant`. FACTO also keeps the tuples whose input features fit 256-element blocks. Authored with Claude Code. Differential Revision: [D128254890](https://our.internmc.facebook.com/intern/diff/D128254890/) **NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D128254890/)! ghstack-source-id: 463657745 --- backends/native/README.md | 6 +- backends/native/test/samples/BUCK | 3 + backends/native/test/samples/linear.py | 19 ++-- backends/native/test/samples/record.py | 17 ++-- backends/native/test/samples/schemes.py | 114 +++++++++++++++++++++--- 5 files changed, 126 insertions(+), 33 deletions(-) diff --git a/backends/native/README.md b/backends/native/README.md index c918183cddc..33e5d73cf24 100644 --- a/backends/native/README.md +++ b/backends/native/README.md @@ -217,9 +217,9 @@ enabled, `native_runner_utils_test` covers their shared runner utilities. Run th with `ctest --test-dir cmake-out -R native_` after `cmake --build cmake-out`. The engines' sample tests judge them on the bundled op samples. Building them runs -the sample generator, which needs `torch`, `torchao` and the FACTO submodule at -`backends/cadence/utils/FACTO`. Those with failing cases are disabled in CTest; -run their binaries directly with +the sample generator, which needs `torch`, `torchao`, `gguf` and the FACTO +submodule at `backends/cadence/utils/FACTO`. Those with failing cases are +disabled in CTest; run their binaries directly with `--packages=/backends/native/test/op_samples`. The `Test Native Engines` CI workflow has separate Linux jobs for the shared diff --git a/backends/native/test/samples/BUCK b/backends/native/test/samples/BUCK index 326d33d26de..d1d4ab455ec 100644 --- a/backends/native/test/samples/BUCK +++ b/backends/native/test/samples/BUCK @@ -17,8 +17,11 @@ fbcode_target( "//caffe2:torch", "//executorch/backends/test/suite/operators:test_operators", "//executorch/exir/native:native", + "//executorch/extension/llm/export:gguf", "//pytorch/ao:torchao", "//pytorch/facto:facto", + "fbsource//third-party/pypi/gguf:gguf", + "fbsource//third-party/pypi/numpy:numpy", ], ) diff --git a/backends/native/test/samples/linear.py b/backends/native/test/samples/linear.py index c1f1dc60e48..9ee902e681c 100644 --- a/backends/native/test/samples/linear.py +++ b/backends/native/test/samples/linear.py @@ -15,6 +15,7 @@ suite_cases, ) from executorch.backends.test.suite.operators import test_linear +from executorch.extension.llm.export.gguf import QK_K from facto.inputgen.argument.type import ArgType from facto.inputgen.specs.model import ConstraintProducer as cp, InPosArg, OutArg, Spec @@ -29,8 +30,9 @@ def _unit(*constraints) -> list: ] -# in_features is a multiple of every scheme's group size, and out_features is -# large enough to fit an error's abs_bias and rel_bias from one sample. +# in_features is a multiple of every torchao scheme's group size and, in some +# tuples, of GGUF's block; out_features is large enough to fit an error's +# abs_bias and rel_bias from one sample. _FACTO_SPEC = Spec( op="linear.default", inspec=[ @@ -98,8 +100,13 @@ def cases() -> list[Case]: ), Case("outlier_channel", torch.nn.Linear(256, 128), outlier), Case("zero_row", torch.nn.Linear(256, 64), zero_row), - # FACTO varies one argument at a time, so only every fifth tuple is kept. - # Its values lie on a 1/8 grid, which makes float32 math exact: these - # cases cover shapes more than rounding. - *facto_cases(_FACTO_SPEC, _facto_case, lambda i, _: i % 5 == 0), + # FACTO varies one argument at a time, so only every fifth tuple is kept, + # plus those whose in_features fit GGUF blocks. Its values lie on a 1/8 + # grid, which makes float32 math exact: these cases cover shapes more + # than rounding. + *facto_cases( + _FACTO_SPEC, + _facto_case, + lambda i, args: i % 5 == 0 or args[1].size(1) % QK_K == 0, + ), ] diff --git a/backends/native/test/samples/record.py b/backends/native/test/samples/record.py index d3565a01667..ef439ce541f 100644 --- a/backends/native/test/samples/record.py +++ b/backends/native/test/samples/record.py @@ -22,7 +22,7 @@ import torch from executorch.backends.native.test.samples.cases import Case, Inputs, outputs -from executorch.backends.native.test.samples.schemes import Scheme +from executorch.backends.native.test.samples.schemes import int8_activations, Scheme from executorch.exir.native import Call, Sample, SampleSet, to_native, write_samples from torch.func import functional_call @@ -51,7 +51,6 @@ def record( dtypes: Iterable[str] = tuple(DTYPES), ) -> None: """Writes no package, removing any at `path`, when the scheme fits no case.""" - act_scheme = scheme.act_quant() programs, golden, precise, golden_act = {}, [], [], [] for case in cases: if not scheme.fits(case.model): @@ -65,9 +64,6 @@ def record( model = copy.deepcopy(case.model).to(dtype).eval() scheme.quantize(model) method = _Method({name: model}) - if act_scheme is not None: - act_model = copy.deepcopy(case.model).to(dtype).eval() - act_scheme.quantize(act_model) state = { k: scheme.decode(v).double() for k, v in method.state_dict().items() } @@ -81,10 +77,11 @@ def record( t.float() if t.is_floating_point() else t for t in outputs(ref) ] precise.append(Sample([Call(name, list(x), ref)])) - if act_scheme is not None: - golden_act.append( - Sample([Call(name, list(x), outputs(act_model(*x)))]) - ) + if scheme.weight_only: + with int8_activations(method): + golden_act.append( + Sample([Call(name, list(x), outputs(method(*x)))]) + ) if not programs: logger.info(f"{path.name}: no case fits the scheme, not written") path.unlink(missing_ok=True) @@ -99,7 +96,7 @@ def record( precise, "float64, weights dequantized, activations unquantized", source ), ) - if act_scheme is not None: + if scheme.weight_only: write_samples( str(path), "golden-int8_act", diff --git a/backends/native/test/samples/schemes.py b/backends/native/test/samples/schemes.py index 82d9c2d8cec..af4183da9b1 100644 --- a/backends/native/test/samples/schemes.py +++ b/backends/native/test/samples/schemes.py @@ -7,9 +7,15 @@ """Quantization schemes. Every op is recorded under every scheme, keeping the cases the scheme fits.""" +import contextlib +import tempfile +from collections.abc import Iterator from dataclasses import dataclass +import numpy as np import torch +from executorch.extension.llm.export.gguf import iter_gguf, QK_K +from gguf import GGML_QUANT_SIZES, GGMLQuantizationType, GGUFWriter from torchao.core.config import AOBaseConfig from torchao.quantization import ( Int8DynamicActivationIntxWeightConfig, @@ -17,12 +23,24 @@ quantize_, ) from torchao.quantization.granularity import PerGroup + +# Private, but it is the input quantize-dequantize that +# Int8DynamicActivationIntxWeightConfig linears run. +from torchao.quantization.quantize_.workflows.intx.intx_unpacked_to_int8_tensor import ( + _apply_int8_act_asym_per_token_quant_dequant, +) from torchao.utils import TorchAOBaseTensor class Scheme: """No quantization.""" + @property + def weight_only(self) -> bool: + """Whether the scheme quantizes only weights; `record` then adds + `golden-int8_act`.""" + return False + def fits(self, model: torch.nn.Module) -> bool: return True @@ -33,11 +51,6 @@ def decode(self, value: torch.Tensor) -> torch.Tensor: """A state tensor of a quantized model as a plain tensor.""" return value - def act_quant(self) -> "Scheme | None": - """This weight-only scheme's weights with int8 dynamic activations, or - None when the scheme is not weight-only.""" - return None - def _fits(model: torch.nn.Module, group: int | None) -> bool: """Whether `model` has an nn.Linear and every one's in_features split into @@ -57,6 +70,10 @@ class Torchao(Scheme): config: AOBaseConfig + @property + def weight_only(self) -> bool: + return isinstance(self.config, IntxWeightOnlyConfig) + def fits(self, model: torch.nn.Module) -> bool: granularity = ( self.config.granularity @@ -73,17 +90,84 @@ def quantize(self, model: torch.nn.Module) -> None: def decode(self, value: torch.Tensor) -> torch.Tensor: return value.dequantize() if isinstance(value, TorchAOBaseTensor) else value - def act_quant(self) -> Scheme | None: - if not isinstance(self.config, IntxWeightOnlyConfig): - return None - return Torchao( - Int8DynamicActivationIntxWeightConfig( - weight_dtype=self.config.weight_dtype, - weight_granularity=self.config.granularity, - weight_mapping_type=self.config.mapping_type, - weight_scale_dtype=self.config.scale_dtype, + +# fp16 super-block scales by byte offset in each block, kept in fp16's normal +# range; q4_k's dmin of 8 d centers random weights near zero. +_GGUF_SUPER_SCALES = {"q4_k": {0: 2**-12, 2: 2**-9}, "q6_k": {208: 2**-12}} + + +def _random_blocks( + ggml_type: str, rows: int, cols: int, rng: np.random.Generator +) -> np.ndarray: + """Valid `ggml_type` blocks for a [rows, cols] weight from random bytes.""" + _, block_bytes = GGML_QUANT_SIZES[GGMLQuantizationType[ggml_type.upper()]] + raw = rng.integers(0, 256, (rows, cols // QK_K, block_bytes), dtype=np.uint8) + for offset, scale in _GGUF_SUPER_SCALES[ggml_type].items(): + raw[..., offset : offset + 2] = np.array([scale], np.float16).view(np.uint8) + return raw.reshape(rows, -1) + + +def _load_gguf(model: torch.nn.Module, ggml_type: str) -> None: + """Writes random `ggml_type` weights for every nn.Linear to a .gguf file and + loads them back as a GGUF checkpoint would be.""" + linears = [m for m in model.modules() if isinstance(m, torch.nn.Linear)] + rng = np.random.default_rng(0) + with tempfile.TemporaryDirectory() as tmp: + path = f"{tmp}/weights.gguf" + writer = GGUFWriter(path, arch="linear") + for i, m in enumerate(linears): + writer.add_tensor( + f"linear.{i}.weight", + _random_blocks(ggml_type, *m.weight.shape, rng), + raw_dtype=GGMLQuantizationType[ggml_type.upper()], + ) + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_tensors_to_file() + writer.close() + for m, (_, weight) in zip(linears, iter_gguf(path), strict=True): + m.weight = torch.nn.Parameter( + weight.to(m.weight.dtype), requires_grad=False ) + + +@dataclass(frozen=True) +class Gguf(Scheme): + """Weight-only GGUF k-quant blocks (`q4_k` or `q6_k`) for every nn.Linear, + loaded from a .gguf file.""" + + ggml_type: str + + @property + def weight_only(self) -> bool: + return True + + def fits(self, model: torch.nn.Module) -> bool: + return _fits(model, QK_K) + + def quantize(self, model: torch.nn.Module) -> None: + _load_gguf(model, self.ggml_type) + + def decode(self, value: torch.Tensor) -> torch.Tensor: + return value.dequantize() if isinstance(value, TorchAOBaseTensor) else value + + +@contextlib.contextmanager +def int8_activations(model: torch.nn.Module) -> Iterator[None]: + """Quantizes every nn.Linear input as torchao's int8 dynamic activation + schemes do: int8, asymmetric, per token.""" + hooks = [ + m.register_forward_pre_hook( + lambda _, args: (_apply_int8_act_asym_per_token_quant_dequant(args[0]),) ) + for m in model.modules() + if isinstance(m, torch.nn.Linear) + ] + try: + yield + finally: + for hook in hooks: + hook.remove() def schemes() -> dict[str, Scheme]: @@ -105,4 +189,6 @@ def schemes() -> dict[str, Scheme]: weight_dtype=torch.int4, weight_granularity=PerGroup(128) ) ), + "gguf_q4_k": Gguf("q4_k"), + "gguf_q6_k": Gguf("q6_k"), } From f35b111c349f98b9ff810e624284d7a123f94921 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:29 -0800 Subject: [PATCH 13/20] [executorch][native] Decode quantized weights in float32 for precise samples Pull Request resolved: https://github.com/pytorch/executorch/pull/24367 `precise` used weights dequantized in the method's dtype, so f16 and bf16 references held already-rounded weights while engines decode in float32. The `Gguf` and `Torchao` schemes now decode their weights in float32 before the float64 math and raise on any other torchao weight. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657760 @exported-using-ghexport Differential Revision: [D128254893](https://our.internmc.facebook.com/intern/diff/D128254893/) --- backends/native/test/samples/BUCK | 1 + backends/native/test/samples/record.py | 15 ++++--- backends/native/test/samples/schemes.py | 20 +++++++-- backends/native/test/samples/test_record.py | 50 ++++++++++++++++++++- 4 files changed, 75 insertions(+), 11 deletions(-) diff --git a/backends/native/test/samples/BUCK b/backends/native/test/samples/BUCK index d1d4ab455ec..6ff24d67abc 100644 --- a/backends/native/test/samples/BUCK +++ b/backends/native/test/samples/BUCK @@ -41,6 +41,7 @@ fbcode_target( ":record", "//caffe2:torch", "//executorch/exir/native:native", + "//pytorch/ao:torchao", ], ) diff --git a/backends/native/test/samples/record.py b/backends/native/test/samples/record.py index ef439ce541f..90a57daa399 100644 --- a/backends/native/test/samples/record.py +++ b/backends/native/test/samples/record.py @@ -8,11 +8,12 @@ A package holds one method per case and dtype under one quantization scheme, and two sample sets over the same inputs: `golden`, the eager outputs, and -`precise`, the same math in float64 with weights dequantized and activations -unquantized, floating outputs stored as float32. Weight-only schemes add -`golden-int8_act`, the eager outputs with the same weights and int8 per-token -dynamic activations. Nothing in it depends on an engine or a device: consumers -judge an engine by its distance from `precise` relative to eager's. +`precise`, the same math in float64 with quantized weights decoded in float32, +whatever the method's dtype, and activations unquantized, floating outputs +stored as float32. Weight-only schemes add `golden-int8_act`, the eager outputs +with the same weights and int8 per-token dynamic activations. Nothing in it +depends on an engine or a device: consumers judge an engine by its distance +from `precise` relative to eager's. """ import copy @@ -93,7 +94,9 @@ def record( str(path), "precise", SampleSet( - precise, "float64, weights dequantized, activations unquantized", source + precise, + "float64, quantized weights decoded in float32, activations unquantized", + source, ), ) if scheme.weight_only: diff --git a/backends/native/test/samples/schemes.py b/backends/native/test/samples/schemes.py index af4183da9b1..386c24bf71f 100644 --- a/backends/native/test/samples/schemes.py +++ b/backends/native/test/samples/schemes.py @@ -14,11 +14,13 @@ import numpy as np import torch -from executorch.extension.llm.export.gguf import iter_gguf, QK_K +from executorch.extension.llm.export.gguf import ExportableGGUFTensor, iter_gguf, QK_K from gguf import GGML_QUANT_SIZES, GGMLQuantizationType, GGUFWriter from torchao.core.config import AOBaseConfig from torchao.quantization import ( + dequantize_affine, Int8DynamicActivationIntxWeightConfig, + IntxUnpackedToInt8Tensor, IntxWeightOnlyConfig, quantize_, ) @@ -48,7 +50,9 @@ def quantize(self, model: torch.nn.Module) -> None: pass def decode(self, value: torch.Tensor) -> torch.Tensor: - """A state tensor of a quantized model as a plain tensor.""" + """A state tensor of a quantized model as a plain tensor, with quantized + weights decoded in float32 rather than the method's dtype, which can + round them.""" return value @@ -88,7 +92,13 @@ def quantize(self, model: torch.nn.Module) -> None: quantize_(model, self.config) def decode(self, value: torch.Tensor) -> torch.Tensor: - return value.dequantize() if isinstance(value, TorchAOBaseTensor) else value + if isinstance(value, IntxUnpackedToInt8Tensor): + return dequantize_affine( + value.qdata, value.block_size, value.scale, value.zero_point, torch.int8 + ) + if isinstance(value, TorchAOBaseTensor): + raise TypeError(f"no float32 decode for {type(value).__name__}") + return value # fp16 super-block scales by byte offset in each block, kept in fp16's normal @@ -149,7 +159,9 @@ def quantize(self, model: torch.nn.Module) -> None: _load_gguf(model, self.ggml_type) def decode(self, value: torch.Tensor) -> torch.Tensor: - return value.dequantize() if isinstance(value, TorchAOBaseTensor) else value + if isinstance(value, ExportableGGUFTensor): + return value.dequantize(torch.float32) + return value @contextlib.contextmanager diff --git a/backends/native/test/samples/test_record.py b/backends/native/test/samples/test_record.py index 086f2a55d9b..1438f648162 100644 --- a/backends/native/test/samples/test_record.py +++ b/backends/native/test/samples/test_record.py @@ -4,6 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import copy import tempfile import unittest from pathlib import Path @@ -13,8 +14,15 @@ import torch.nn.functional as F from executorch.backends.native.test.samples.cases import Case, draws, suite_cases from executorch.backends.native.test.samples.record import record -from executorch.backends.native.test.samples.schemes import Scheme, schemes +from executorch.backends.native.test.samples.schemes import ( + Gguf, + Scheme, + schemes, + Torchao, +) from executorch.exir.native import read_samples +from torchao.quantization import IntxWeightOnlyConfig, quantize_ +from torchao.quantization.granularity import PerGroup class RecordTest(unittest.TestCase): @@ -107,3 +115,43 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: record([case], Scheme(), path, dtypes=["f32"]) precise = read_samples(str(path))["precise"].samples self.assertEqual(precise[0].calls[0].outputs[0].dtype, torch.int64) + + def test_precise_decodes_gguf_weights_whatever_the_method_dtype(self) -> None: + # Halves are exact in every recorded dtype and the methods share their + # GGUF blocks, so their precise outputs must match. + x = torch.randint(-8, 9, (3, 256)) / 2 + model = torch.nn.Linear(256, 8, bias=False) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "linear.ptn" + record([Case("linear", model, [(x,)])], Gguf("q6_k"), path) + samples = read_samples(str(path))["precise"].samples + outputs = {s.calls[0].method: s.calls[0].outputs[0] for s in samples} + for dtype in ("f16", "bf16"): + torch.testing.assert_close( + outputs[f"linear_{dtype}"], outputs["linear_f32"], rtol=0, atol=0 + ) + + def test_precise_decodes_intx_weights_in_float32(self) -> None: + # A code times its scale can need more bits than bf16 holds, so decoding + # in the method's dtype would round the weights. + x = torch.randint(-8, 9, (3, 64)) / 2 + model = torch.nn.Linear(64, 8, bias=False) + config = IntxWeightOnlyConfig(weight_dtype=torch.int4, granularity=PerGroup(32)) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "linear.ptn" + record([Case("linear", model, [(x,)])], Torchao(config), path) + samples = read_samples(str(path))["precise"].samples + outputs = {s.calls[0].method: s.calls[0].outputs[0] for s in samples} + bf16 = copy.deepcopy(model).to(torch.bfloat16) + quantize_(bf16, config) + w = bf16.weight + codes = ( + w.qdata.double().unflatten(1, (-1, 32)) - w.zero_point.double()[..., None] + ) + weight = (codes * w.scale.double()[..., None]).flatten(1) + torch.testing.assert_close( + outputs["linear_bf16"], + torch.nn.functional.linear(x.double(), weight).float(), + rtol=0, + atol=0, + ) From 144eba81a2c72328a79c96b22f1c9677f029948d Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:29 -0800 Subject: [PATCH 14/20] [executorch][native] Cover negative, rounded and subnormal GGUF super-block scales in the op samples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pull Request resolved: https://github.com/pytorch/executorch/pull/24368 GGUF samples used fixed positive power-of-two super-block scales, so engines' decoders never saw negative, rounded or subnormal ones. A model's i-th `nn.Linear` now takes the i-th scale family, and a new `scale_families` case gives each family its own output. A zero-d family waits behind a TODO. With q4_k's d at 0 every weight is -dmin·m, so a token's 32 outputs share one activation-quantization error (correlation about 0.74) and `OutputCheck`'s bias standard error, which assumes independent outputs, comes out about 4.9 times too small; with q6_k's d at 0 every weight is zero and the outputs are just the bias. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657756 @exported-using-ghexport Differential Revision: [D128254896](https://our.internmc.facebook.com/intern/diff/D128254896/) --- backends/native/test/samples/linear.py | 18 ++++++++ backends/native/test/samples/schemes.py | 46 +++++++++++++++++---- backends/native/test/samples/test_record.py | 11 +++++ 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/backends/native/test/samples/linear.py b/backends/native/test/samples/linear.py index 9ee902e681c..9beea6408fa 100644 --- a/backends/native/test/samples/linear.py +++ b/backends/native/test/samples/linear.py @@ -14,6 +14,7 @@ facto_cases, suite_cases, ) +from executorch.backends.native.test.samples.schemes import GGUF_SUPER_SCALES from executorch.backends.test.suite.operators import test_linear from executorch.extension.llm.export.gguf import QK_K from facto.inputgen.argument.type import ArgType @@ -78,6 +79,17 @@ def _facto_case(name: str, x, weight, bias) -> Case: return Case(name, model, [(x,)]) +class _Parallel(torch.nn.Module): + """nn.Linears on one input, each giving its own output.""" + + def __init__(self, *linears: torch.nn.Linear) -> None: + super().__init__() + self.linears = torch.nn.ModuleList(linears) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, ...]: + return tuple(m(x) for m in self.linears) + + def cases() -> list[Case]: suite = suite_cases(test_linear, samples=4) torch.manual_seed(0) @@ -100,6 +112,12 @@ def cases() -> list[Case]: ), Case("outlier_channel", torch.nn.Linear(256, 128), outlier), Case("zero_row", torch.nn.Linear(256, 64), zero_row), + # One output per family of GGUF super-block scales. + Case( + "scale_families", + _Parallel(*(torch.nn.Linear(256, 32) for _ in GGUF_SUPER_SCALES["q4_k"])), + draws(2, (4, 256)), + ), # FACTO varies one argument at a time, so only every fifth tuple is kept, # plus those whose in_features fit GGUF blocks. Its values lie on a 1/8 # grid, which makes float32 math exact: these cases cover shapes more diff --git a/backends/native/test/samples/schemes.py b/backends/native/test/samples/schemes.py index 386c24bf71f..c0c2f3d1fe8 100644 --- a/backends/native/test/samples/schemes.py +++ b/backends/native/test/samples/schemes.py @@ -101,18 +101,40 @@ def decode(self, value: torch.Tensor) -> torch.Tensor: return value -# fp16 super-block scales by byte offset in each block, kept in fp16's normal -# range; q4_k's dmin of 8 d centers random weights near zero. -_GGUF_SUPER_SCALES = {"q4_k": {0: 2**-12, 2: 2**-9}, "q6_k": {208: 2**-12}} +# Families of fp16 super-block scales by byte offset in each block: powers of +# two, where q4_k's dmin of 8 d centers random weights near zero, then negative, +# rounded and subnormal ones. A model's i-th nn.Linear takes the i-th family, so +# one nn.Linear per family isolates each in its own output. +# TODO: Add a zero-d family once OutputCheck's bias standard error stops +# assuming independent outputs. +GGUF_SUPER_SCALES = { + "q4_k": [ + {0: 2**-12, 2: 2**-9}, + {0: -(2**-12), 2: -(2**-9)}, + {0: 3.1e-4, 2: 2.3e-3}, + {0: 3 * 2**-21, 2: 3 * 2**-18}, + ], + "q6_k": [ + {208: 2**-12}, + {208: -(2**-12)}, + {208: 3.1e-4}, + {208: 3 * 2**-21}, + ], +} def _random_blocks( - ggml_type: str, rows: int, cols: int, rng: np.random.Generator + ggml_type: str, + rows: int, + cols: int, + scales: dict[int, float], + rng: np.random.Generator, ) -> np.ndarray: - """Valid `ggml_type` blocks for a [rows, cols] weight from random bytes.""" + """Valid `ggml_type` blocks for a [rows, cols] weight from random bytes and + the given super-block scales.""" _, block_bytes = GGML_QUANT_SIZES[GGMLQuantizationType[ggml_type.upper()]] raw = rng.integers(0, 256, (rows, cols // QK_K, block_bytes), dtype=np.uint8) - for offset, scale in _GGUF_SUPER_SCALES[ggml_type].items(): + for offset, scale in scales.items(): raw[..., offset : offset + 2] = np.array([scale], np.float16).view(np.uint8) return raw.reshape(rows, -1) @@ -125,10 +147,15 @@ def _load_gguf(model: torch.nn.Module, ggml_type: str) -> None: with tempfile.TemporaryDirectory() as tmp: path = f"{tmp}/weights.gguf" writer = GGUFWriter(path, arch="linear") - for i, m in enumerate(linears): + families = GGUF_SUPER_SCALES[ggml_type] + if len(linears) > len(families): + raise ValueError( + f"{len(linears)} nn.Linears, {len(families)} {ggml_type} scale families" + ) + for i, (m, scales) in enumerate(zip(linears, families)): writer.add_tensor( f"linear.{i}.weight", - _random_blocks(ggml_type, *m.weight.shape, rng), + _random_blocks(ggml_type, *m.weight.shape, scales, rng), raw_dtype=GGMLQuantizationType[ggml_type.upper()], ) writer.write_header_to_file() @@ -153,7 +180,8 @@ def weight_only(self) -> bool: return True def fits(self, model: torch.nn.Module) -> bool: - return _fits(model, QK_K) + linears = sum(isinstance(m, torch.nn.Linear) for m in model.modules()) + return _fits(model, QK_K) and linears <= len(GGUF_SUPER_SCALES[self.ggml_type]) def quantize(self, model: torch.nn.Module) -> None: _load_gguf(model, self.ggml_type) diff --git a/backends/native/test/samples/test_record.py b/backends/native/test/samples/test_record.py index 1438f648162..fd5d28d3b2f 100644 --- a/backends/native/test/samples/test_record.py +++ b/backends/native/test/samples/test_record.py @@ -16,6 +16,7 @@ from executorch.backends.native.test.samples.record import record from executorch.backends.native.test.samples.schemes import ( Gguf, + GGUF_SUPER_SCALES, Scheme, schemes, Torchao, @@ -131,6 +132,16 @@ def test_precise_decodes_gguf_weights_whatever_the_method_dtype(self) -> None: outputs[f"linear_{dtype}"], outputs["linear_f32"], rtol=0, atol=0 ) + def test_gguf_fits_no_more_linears_than_scale_families(self) -> None: + def linears(count: int) -> torch.nn.Module: + return torch.nn.Sequential( + *(torch.nn.Linear(256, 256) for _ in range(count)) + ) + + families = len(GGUF_SUPER_SCALES["q4_k"]) + self.assertTrue(Gguf("q4_k").fits(linears(families))) + self.assertFalse(Gguf("q4_k").fits(linears(families + 1))) + def test_precise_decodes_intx_weights_in_float32(self) -> None: # A code times its scale can need more bits than bf16 holds, so decoding # in the method's dtype would round the weights. From 515fc2efde581f0f4c13b97969359d8c91efb6e7 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:30 -0800 Subject: [PATCH 15/20] [executorch][vulkan] Accept 1-D activations in 8da4w linear Pull Request resolved: https://github.com/pytorch/executorch/pull/24369 `linear_dq8ca_q4gsw` failed on rank-1 activations: `is_2d_matrix` rejected them, and `get_quantized_input_num_blocks` read the size at `ndim - 2`. Both now treat a 1-D tensor as a single row. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657751 @exported-using-ghexport Differential Revision: [D128254882](https://our.internmc.facebook.com/intern/diff/D128254882/) --- .../vulkan/runtime/graph/ComputeGraph.cpp | 4 +- backends/vulkan/runtime/graph/ComputeGraph.h | 6 +- .../graph/ops/impl/QuantizeDequantize.cpp | 7 +- .../vulkan/test/vulkan_compute_api_test.cpp | 78 +++++++++++++++++++ 4 files changed, 86 insertions(+), 9 deletions(-) diff --git a/backends/vulkan/runtime/graph/ComputeGraph.cpp b/backends/vulkan/runtime/graph/ComputeGraph.cpp index a6b3c4f6347..e3997b26988 100644 --- a/backends/vulkan/runtime/graph/ComputeGraph.cpp +++ b/backends/vulkan/runtime/graph/ComputeGraph.cpp @@ -426,10 +426,10 @@ bool ComputeGraph::is_standard_channels_packed_texture_tensor( bool ComputeGraph::is_2d_matrix(const ValueRef idx) const { std::vector sizes = sizes_of(idx); const size_t ndim = sizes.size(); - if (sizes.size() < 2) { + if (ndim == 0) { return false; } - if (sizes.size() == 2) { + if (ndim <= 2) { return true; } diff --git a/backends/vulkan/runtime/graph/ComputeGraph.h b/backends/vulkan/runtime/graph/ComputeGraph.h index 6b78d8b50fb..eb0a0e168d2 100644 --- a/backends/vulkan/runtime/graph/ComputeGraph.h +++ b/backends/vulkan/runtime/graph/ComputeGraph.h @@ -440,9 +440,9 @@ class ComputeGraph final { bool is_standard_channels_packed_texture_tensor(const ValueRef idx) const; /* - * Checks that the value at `idx` is either a 2D tensor, or if the tensor has - * more than 2 dims, the outermost dims have size of 1, i.e. can be squeezed - * to be a 2D tensor. + * Checks that the value at `idx` is a 1D tensor (a single row), a 2D tensor, + * or a tensor whose outermost dims have size of 1, i.e. can be squeezed to + * be a 2D tensor. */ bool is_2d_matrix(const ValueRef idx) const; diff --git a/backends/vulkan/runtime/graph/ops/impl/QuantizeDequantize.cpp b/backends/vulkan/runtime/graph/ops/impl/QuantizeDequantize.cpp index 7cf03d98a94..5834f2551e1 100644 --- a/backends/vulkan/runtime/graph/ops/impl/QuantizeDequantize.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/QuantizeDequantize.cpp @@ -32,11 +32,10 @@ bool is_gemv(ComputeGraph* graph, const ValueRef& fp_input) { std::tuple get_quantized_input_num_blocks( ComputeGraph& graph, const ValueRef input) { - std::vector input_sizes = graph.sizes_of(input); - const int64_t ndim = graph.dim_of(input); + const std::vector input_sizes = graph.sizes_of(input); - const int64_t M = input_sizes.at(ndim - 2); - const int64_t K = input_sizes.at(ndim - 1); + const int64_t M = utils::val_at(-2, input_sizes); + const int64_t K = utils::val_at(-1, input_sizes); const int64_t num_blocks_M = utils::div_up(M, int64_t(4)); const int64_t num_blocks_K = utils::div_up(K, int64_t(4)); diff --git a/backends/vulkan/test/vulkan_compute_api_test.cpp b/backends/vulkan/test/vulkan_compute_api_test.cpp index 0374505d0bb..b087b0e4baa 100644 --- a/backends/vulkan/test/vulkan_compute_api_test.cpp +++ b/backends/vulkan/test/vulkan_compute_api_test.cpp @@ -2465,6 +2465,84 @@ TEST(VulkanComputeGraphTest, choose_qparams_handles_dynamic_row_counts) { } } +TEST(VulkanComputeGraphTest, choose_qparams_affine_handles_1d_input) { + constexpr int64_t kK = 128; + + ComputeGraph graph(GraphConfig{}); + const IOValueRef input = + graph.add_input_tensor({kK}, vkapi::kFloat, utils::kBuffer); + const ValueRef scales = graph.add_tensor( + {1}, vkapi::kFloat, utils::kTexture3D, utils::kWidthPacked); + const ValueRef zero_points = graph.add_tensor( + {1}, vkapi::kChar, utils::kTexture3D, utils::kWidthPacked); + + VK_GET_OP_FN("torchao.choose_qparams_affine.default") + (graph, + {input.value, + graph.add_none(), // mapping_type + graph.add_scalar_list({kK}), + graph.add_none(), // target_dtype + graph.add_scalar(-128), + graph.add_scalar(127), + graph.add_none(), // eps + graph.add_none(), // scale_dtype + graph.add_none(), // zero_point_dtype + graph.add_value_list({scales, zero_points})}); + + const ValueRef scales_staging = graph.set_output_tensor(scales); + const ValueRef zero_points_staging = graph.set_output_tensor(zero_points); + + graph.prepare(); + graph.prepack(); + + // The row's maximum is its last element, so the whole row must be reduced. + std::vector input_data(kK); + for (int64_t k = 0; k < kK; ++k) { + input_data[k] = 0.5f * float(k); + } + graph.maybe_cast_and_copy_into_staging( + input.staging, input_data.data(), input_data.size(), vkapi::kFloat); + + graph.execute(); + + float scale = 0; + int8_t zero_point = 0; + graph.maybe_cast_and_copy_from_staging( + scales_staging, &scale, 1, vkapi::kFloat); + graph.maybe_cast_and_copy_from_staging( + zero_points_staging, &zero_point, 1, vkapi::kChar); + EXPECT_NEAR(scale, input_data.back() / 255.0f, 1e-6f); + EXPECT_EQ(zero_point, -128); +} + +TEST(VulkanComputeGraphTest, quantize_and_pack_accepts_1d_input) { + constexpr int64_t kK = 128; + + ComputeGraph graph(GraphConfig{}); + const IOValueRef input = + graph.add_input_tensor({kK}, vkapi::kFloat, utils::kBuffer); + const ValueRef scales = graph.add_tensor( + {1}, vkapi::kFloat, utils::kTexture3D, utils::kWidthPacked); + const ValueRef zero_points = graph.add_tensor( + {1}, vkapi::kChar, utils::kTexture3D, utils::kWidthPacked); + const ValueRef packed_input = graph.add_tensor( + {1, kK}, vkapi::kInt8x4, utils::kBuffer, utils::kPackedInt8_4H4W); + const ValueRef input_sums = + graph.add_tensor({4}, vkapi::kInt, utils::kBuffer, utils::kWidthPacked); + const QuantizationConfig input_quant_config( + 8, kPerChannel, {1, kK}, false, true); + + EXPECT_NO_THROW(add_quantize_and_pack_4h4w_with_group_sums_node( + graph, + input_quant_config, + input.value, + input_sums, + scales, + zero_points, + packed_input, + graph.add_scalar(kK))); +} + void test_quantize_and_pack_handles_dynamic_row_counts( const int64_t group_size_value, const utils::uvec3& expected_local_wg_size) { From 2b2ec0e47b213f29539923ebd9bb78dc944edc8c Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:31 -0800 Subject: [PATCH 16/20] [executorch][vulkan] Fix dq8ca q4gsw weight sums stride when N % 8 is 1 to 4 Pull Request resolved: https://github.com/pytorch/executorch/pull/24370 `linear_dq8ca_q4gsw_tiled` strode each group's weight sums by `div_up(N, 4)` texels, but the sums are laid out `{groups, align_up(N, 8)}`, as both the ET-VK AOT pass (`compute_per_group_sums`) and Native's `FuseQuantizedLinear` write them. When `div_up(N, 4)` is odd, every group after the first read another row's sums; the shader now strides by `2 * div_up(N, 8)`, which equals the old stride for every other N. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657747 @exported-using-ghexport Differential Revision: [D128254883](https://our.internmc.facebook.com/intern/diff/D128254883/) --- .../gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl index 5acab8e9a12..d29167e5c9a 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/gemm/linear/qw/dq8ca_q4gsw/linear_dq8ca_q4gsw_tiled.glsl @@ -130,7 +130,8 @@ void main() { } load_weight_scales_tile_for_group(weight_scales_tile, n4, group_i, N4); - load_weight_sums_tile_for_group(weight_sums_tile, n4, group_i, N4); + // Weight sums rows are padded to a multiple of 8. + load_weight_sums_tile_for_group(weight_sums_tile, n4, group_i, mul_2(N8)); load_int8_input_sums_tile_for_group(int8_input_sums_tile, m4, group_i, M4); accumulate_out_tile_with_int_accum_from_int4_weights( From cfcee439c4e1b4a8a7384be1dc6d9c7f505bfad6 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:32 -0800 Subject: [PATCH 17/20] [executorch][native-vk] Note the int4 weight-only N % 4 and rank-4 linear gaps Pull Request resolved: https://github.com/pytorch/executorch/pull/24371 The Vulkan engine leaves int4 weight-only linears unlowered when N isn't a multiple of 4, so op-sample cases with N of 1 and 43 fail. It also computes a linear on a rank-4 input wrong, while rank-3 inputs pass. `TODO.md` records both, with the fix for the first and where to look for the second. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657741 @exported-using-ghexport Differential Revision: [D128254884](https://our.internmc.facebook.com/intern/diff/D128254884/) --- backends/native/runtime/vulkan/TODO.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/backends/native/runtime/vulkan/TODO.md b/backends/native/runtime/vulkan/TODO.md index 2b6b1de54df..f4c8d3bd7ad 100644 --- a/backends/native/runtime/vulkan/TODO.md +++ b/backends/native/runtime/vulkan/TODO.md @@ -14,3 +14,23 @@ HuggingFace Llama 3.2 1B 8da4w, 2K export: peak GPU memory is 2794 MiB, against 2265 MiB with buffer SDPA disabled and 2263 MiB for the ET-VK `.pte`. The +528 MiB comes from enabling buffer SDPA. Give these temporaries lifetimes in the memory plan, measure again, and attribute whatever remains. + +## Lower int4 weight-only linear for any N + +`FuseQuantizedLinear` leaves an int4 weight-only linear unlowered when N is not +a multiple of 4, to match `VK_CHECK_COND(N % 4 == 0)` in ET-VK's +`prepack_q4_w_4x8_nc_buffer`, and the engine then rejects the quantized weight. +Round N up to a multiple of 4 there (the pack shader already fills n >= N with +zeros) and mask the partial last tile on store. Until then Vulkan fails the op +sample tests `linear_int4wo_*.linear_feature_sizes_2` and +`linear_int4wo_*.linear_unbatched_0`, where N is 1 and 43. + +## Rank-4 linear input + +Vulkan returns wrong outputs for the op sample test +`linear_float.linear_leading_batch_1`, an `nn.Linear(67, 43)` on a +(2, 4, 8, 67) input: in f32 the fitted rel_bias is -0.48, and f16 returns +values up to 6.1e4. Both are consistent with only one of the two outer batches +being written. The rank-3 inputs (4, 8, 67) and (2, 7, 256) pass. Find where +the outer dimension is lost: the engine's linear lowering, or ET-VK's linear on +a rank-4 tensor. From 69773cc1eb3e83c301ea96c8e33ff6c234851383 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:32 -0800 Subject: [PATCH 18/20] [executorch][vulkan] Round f16 image stores to nearest even Pull Request resolved: https://github.com/pytorch/executorch/pull/24372 Vulkan lets the device choose how stores to rgba16f images round, and devices that round toward zero bias f16 outputs; shader float controls don't govern that conversion. `gen_vulkan_spv.py` now wraps `imageStore` on writable rgba16f images in a round-to-nearest-even helper, and six shaders that declared such images by hand, two of them test shaders, now use the codegen. On devices that round toward zero, stores of 65520 and above now give inf, as the CPU conversion does, instead of 65504. The four texture shaders that already call `round_to_half_rte` on their outputs (`binary_scalar_texture`, `full_texture`, `reduce`, `scalar_tensor`) now round twice, with identical results; removing those calls is a follow-up. `aten._to_copy` from f32 to f16 textures is a `vkCmdBlitImage`, which the codegen doesn't reach, so `test_to_copy` keeps its one-ulp tolerance. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657749 @exported-using-ghexport Differential Revision: [D128254886](https://our.internmc.facebook.com/intern/diff/D128254886/) --- backends/vulkan/runtime/gen_vulkan_spv.py | 37 +++++++++++ .../conv2d_dw_prepack_weights.glsl | 2 +- .../pack_weights/conv2d_prepack_weights.glsl | 2 +- .../conv_transpose2d_prepack_weights.glsl | 2 +- .../graph/ops/glsl/repeat_channel.glsl | 2 +- .../glsl/binary_op_nobroadcast__test.glsl | 4 +- .../vulkan/test/glsl/fill_texture__test.glsl | 2 +- .../test/spv_codegen/test_gen_vulkan_spv.py | 16 +++++ .../vulkan/test/vulkan_compute_api_test.cpp | 65 +++++++++++++++++++ 9 files changed, 124 insertions(+), 8 deletions(-) diff --git a/backends/vulkan/runtime/gen_vulkan_spv.py b/backends/vulkan/runtime/gen_vulkan_spv.py index 5701248266e..76999cb0995 100644 --- a/backends/vulkan/runtime/gen_vulkan_spv.py +++ b/backends/vulkan/runtime/gen_vulkan_spv.py @@ -355,6 +355,38 @@ def layout_declare_buffer( return out_str +# Vulkan leaves the rounding of a store to an rgba16f image to the device, and +# some devices round toward zero. Rounding to the nearest f16, ties to even, in +# the shader makes the store exact. Integer texels are stored as is, but stores +# to rgba32f images would be rounded too, so a shader may not write both. +ROUND_F16_IMAGE_STORES = """ +#ifdef WRITES_RGBA32F_IMAGE +#error "A shader cannot write both rgba16f and rgba32f images" +#endif +#ifndef WRITES_RGBA16F_IMAGE +#define WRITES_RGBA16F_IMAGE +vec4 round_to_f16(const vec4 v) { + const uvec4 u = floatBitsToUint(v); + // Keep 10 mantissa bits: add just under half an f16 ulp, plus one when the + // kept lowest bit is odd so ties go to even, then clear the 13 dropped bits. + vec4 r = uintBitsToFloat((u + 0xFFFu + ((u >> 13) & 1u)) & ~0x1FFFu); + // Below the smallest normal f16, round to a multiple of 2^-24. + r = mix(r, roundEven(v * 16777216.0) * 5.9604644775390625e-8, lessThan(abs(v), vec4(6.103515625e-5))); + r = mix(r, uintBitsToFloat((u & 0x80000000u) | 0x7F800000u), greaterThan(abs(r), vec4(65504.0))); + return mix(r, v, isnan(v)); +} +ivec4 round_to_f16(const ivec4 v) { return v; } +uvec4 round_to_f16(const uvec4 v) { return v; } +#define imageStore(image, pos, texel) imageStore(image, pos, round_to_f16(texel)) +#endif""" + +WRITES_RGBA32F_IMAGE = """ +#ifdef WRITES_RGBA16F_IMAGE +#error "A shader cannot write both rgba16f and rgba32f images" +#endif +#define WRITES_RGBA32F_IMAGE""" + + def layout_declare_image( slot: Union[int, List[int]], access_type: str, @@ -367,6 +399,11 @@ def layout_declare_image( image_type = TYPE_MAPPINGS["IMAGE_T"][image_ndim][dtype] ret_str = f"layout(set = 0, binding = {get_slot_val(slot)}, {image_format}) uniform {precision} restrict {get_access_qualifier(access_type)} {image_type} {var_name};" + if access_type.lower() != "r": + if image_format == "rgba16f": + ret_str += ROUND_F16_IMAGE_STORES + elif image_format == "rgba32f": + ret_str += WRITES_RGBA32F_IMAGE if isinstance(slot, list): slot[0] = slot[0] + 1 diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.glsl index 4707bcf9f9c..4a16ee05d42 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_dw_prepack_weights.glsl @@ -21,7 +21,7 @@ ${define_required_extensions("buffer", BUF_DTYPE)} layout(std430) buffer; -layout(set = 0, binding = 0, ${IMAGE_FORMAT[DTYPE]}) uniform PRECISION restrict writeonly ${IMAGE_T[2][DTYPE]} image_out; +${layout_declare_tensor(0, "w", "image_out", DTYPE, "texture2d")} layout(set = 0, binding = 1) buffer PRECISION restrict readonly Buffer { BUF_T buffer_in[]; }; diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.glsl index b5bec8fca0b..95c8cc3c4ee 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/pack_weights/conv2d_prepack_weights.glsl @@ -29,7 +29,7 @@ $if OUTPUT_BUFFER: f16vec4 buffer_out[]; }; $else: - layout(set = 0, binding = 0, ${IMAGE_FORMAT[DTYPE]}) uniform PRECISION restrict writeonly ${IMAGE_T[2][DTYPE]} image_out; + ${layout_declare_tensor(0, "w", "image_out", DTYPE, "texture2d")} layout(set = 0, binding = 1) buffer PRECISION restrict readonly Buffer { BUF_T buffer_in[]; }; diff --git a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.glsl b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.glsl index 65b998a9c5b..4a47892375e 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/convolution/conv2d/transposed/pack_weights/conv_transpose2d_prepack_weights.glsl @@ -21,7 +21,7 @@ ${define_required_extensions("buffer", BUF_DTYPE)} layout(std430) buffer; -layout(set = 0, binding = 0, ${IMAGE_FORMAT[DTYPE]}) uniform PRECISION restrict writeonly ${IMAGE_T[2][DTYPE]} image_out; +${layout_declare_tensor(0, "w", "image_out", DTYPE, "texture2d")} layout(set = 0, binding = 1) buffer PRECISION restrict readonly Buffer { BUF_T buffer_in[]; }; diff --git a/backends/vulkan/runtime/graph/ops/glsl/repeat_channel.glsl b/backends/vulkan/runtime/graph/ops/glsl/repeat_channel.glsl index 42c7f86aea8..96e9f7aac4c 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/repeat_channel.glsl +++ b/backends/vulkan/runtime/graph/ops/glsl/repeat_channel.glsl @@ -16,7 +16,7 @@ layout(std430) buffer; #include "indexing_utils.h" -layout(set = 0, binding = 0, ${IMAGE_FORMAT[DTYPE]}) uniform PRECISION restrict writeonly ${IMAGE_T[NDIM][DTYPE]} image_out; +${layout_declare_image(0, "w", "image_out", DTYPE, image_ndim=NDIM)} layout(set = 0, binding = 1) uniform PRECISION sampler3D image_in; layout(set = 0, binding = 2) uniform PRECISION restrict RepeatArgs { diff --git a/backends/vulkan/test/glsl/binary_op_nobroadcast__test.glsl b/backends/vulkan/test/glsl/binary_op_nobroadcast__test.glsl index 7f72ac58972..5fcd1513c4c 100644 --- a/backends/vulkan/test/glsl/binary_op_nobroadcast__test.glsl +++ b/backends/vulkan/test/glsl/binary_op_nobroadcast__test.glsl @@ -14,9 +14,7 @@ layout(std430) buffer; -// clang-format off -layout(set = 0, binding = 0, ${IMAGE_FORMAT[DTYPE]}) uniform PRECISION restrict writeonly image3D image_out; -// clang-format on +${layout_declare_image(0, "w", "image_out", DTYPE)} layout(set = 0, binding = 1) uniform PRECISION sampler3D image_in; layout(set = 0, binding = 2) uniform PRECISION sampler3D image_other; diff --git a/backends/vulkan/test/glsl/fill_texture__test.glsl b/backends/vulkan/test/glsl/fill_texture__test.glsl index 76c630de55e..637c12fbb51 100644 --- a/backends/vulkan/test/glsl/fill_texture__test.glsl +++ b/backends/vulkan/test/glsl/fill_texture__test.glsl @@ -12,7 +12,7 @@ layout(std430) buffer; -layout(set = 0, binding = 0, ${IMAGE_FORMAT[DTYPE]}) uniform PRECISION restrict writeonly ${IMAGE_T[NDIM][DTYPE]} uOutput; +${layout_declare_image(0, "w", "uOutput", DTYPE, image_ndim=NDIM)} layout(set = 0, binding = 1) uniform PRECISION restrict Block { ivec3 size; int fill; diff --git a/backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py b/backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py index aef464dbf38..e569d6fe27d 100644 --- a/backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py +++ b/backends/vulkan/test/spv_codegen/test_gen_vulkan_spv.py @@ -13,6 +13,22 @@ from runtime import gen_vulkan_spv +class TestF16ImageStores(unittest.TestCase): + def test_writable_rgba16f_images_round_stores(self) -> None: + for access in ("w", "rw"): + with self.subTest(access=access): + declaration = gen_vulkan_spv.layout_declare_image( + 0, access, "t_out", "half" + ) + self.assertIn("round_to_f16(texel)", declaration) + + def test_other_images_store_unchanged(self) -> None: + for access, dtype in (("r", "half"), ("w", "float"), ("w", "int32")): + with self.subTest(access=access, dtype=dtype): + declaration = gen_vulkan_spv.layout_declare_image(0, access, "t", dtype) + self.assertNotIn("round_to_f16", declaration) + + class TestNestedShaderSources(unittest.TestCase): def _write_fake_compiler( self, directory: Path, assertions: str = "" diff --git a/backends/vulkan/test/vulkan_compute_api_test.cpp b/backends/vulkan/test/vulkan_compute_api_test.cpp index b087b0e4baa..7f4ef389040 100644 --- a/backends/vulkan/test/vulkan_compute_api_test.cpp +++ b/backends/vulkan/test/vulkan_compute_api_test.cpp @@ -1202,6 +1202,71 @@ TEST_F(VulkanComputeAPITest, spec_var_shader_test) { } } +TEST_F(VulkanComputeAPITest, half_texture_store_rounds_to_nearest_even) { + // Ties, their neighbors, f16 subnormals and values at and past the largest + // f16, where rounding toward zero differs from rounding to nearest even. + const std::vector values = { + 1.0f + 0x1p-11f, + 1.0f + 0x3p-11f, + -(1.0f + 0x3p-11f), + std::nextafter(1.0f + 0x1p-11f, 0.0f), + std::nextafter(1.0f + 0x1p-11f, 2.0f), + 0x1p-25f, + 0x3p-25f, + std::nextafter(0x1p-25f, 0.0f), + 0x1.ffcp-15f, + -0x1.0p-24f, + 65504.0f, + 65519.0f, + 65520.0f, + -65520.0f, + 1e6f, + -1e6f, + }; + + struct Params final { + utils::ivec3 size; + int32_t fill; + utils::vec4 values; + }; + + for (size_t i = 0; i < values.size(); i += 4) { + vTensor a( + context(), + {4, 1, 1}, + vkapi::kHalf, + utils::kTexture3D, + utils::kChannelsPacked); + Params block{ + {1, 1, 1}, + 0, + {values[i], values[i + 1], values[i + 2], values[i + 3]}, + }; + ParamsBuffer params(context(), block); + + vkapi::PipelineBarrier pipeline_barrier{}; + context()->submit_compute_job( + VK_KERNEL(fill_texture__test_half), + pipeline_barrier, + GlobalWorkGrid({1u, 1u, 1u}, kExplicitWorkGrid), + LocalWorkGroup(1u, 1u, 1u), + {}, + VK_NULL_HANDLE, + 0, + a.image( + pipeline_barrier, + vkapi::PipelineStage::COMPUTE, + vkapi::MemoryAccessType::WRITE), + params.buffer()); + + const std::vector out = extract_vtensor(a); + for (size_t j = 0; j < 4; ++j) { + EXPECT_EQ(out[j], float(executorch::aten::Half(values[i + j]))) + << "input " << values[i + j]; + } + } +} + TEST_F(VulkanComputeAPITest, update_params_between_submit) { context()->set_cmd(/*reusable = */ true); std::vector sizes = {4, 4, 2}; From 3b3c9f48012fd90410b833a270faf189fec4793c Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:33 -0800 Subject: [PATCH 19/20] [executorch][native] Run the ggml CPU act_quant sample test in CI Pull Request resolved: https://github.com/pytorch/executorch/pull/24373 `ggml_sample_test_act_quant` (CPU) is the one sample test that passes every package, so it leaves the manual set and runs in internal CI. The OSS ggml job now runs it as `native_ggml_sample_test_act_quant`, along with `native_output_check_test`. The rest stay manual with a TODO: plain `ggml_sample_test` fails the weight-only packages by design, and the Vulkan, CUDA and HIP ones have gaps. The OSS ggml job installs `gguf` for the op sample generator, and the workflow also runs on changes to FACTO and `test_linear.py`, which the samples draw from. Authored with Claude Code. @bypass-github-export-checks @bypass-github-pytorch-ci-checks @bypass-github-executorch-ci-checks ghstack-source-id: 463657737 @exported-using-ghexport Differential Revision: [D128668496](https://our.internmc.facebook.com/intern/diff/D128668496/) --- .ci/scripts/test-native.sh | 5 +++-- .github/workflows/test-native.yml | 9 ++++++--- backends/native/test/CMakeLists.txt | 6 +++--- backends/native/test/runtime/ggml/BUCK | 4 ++-- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.ci/scripts/test-native.sh b/.ci/scripts/test-native.sh index 98139a453e2..fadbfe75840 100644 --- a/.ci/scripts/test-native.sh +++ b/.ci/scripts/test-native.sh @@ -36,8 +36,9 @@ case "${component}" in -DEXECUTORCH_BUILD_NATIVE_VISION_RUNNER=ON -DGGML_NATIVE=OFF ) - targets+=(run_hf_llm_ggml native_ggml_engine_test native_llm_runner_utils_test run_classification_ggml_cpu) - test_regex='^native_(ggml_engine|llm_runner_utils)_test$' + # TODO: run the other sample tests once they pass. + targets+=(run_hf_llm_ggml native_ggml_engine_test native_llm_runner_utils_test run_classification_ggml_cpu native_output_check_test native_ggml_sample_test) + test_regex='^native_(ggml_engine|llm_runner_utils|output_check)_test$|^native_ggml_sample_test_act_quant$' ;; vulkan) cmake_args+=( diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index c7a7aae2f0c..71d051f1679 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -17,6 +17,8 @@ on: - backends/cpu/CMakeLists.txt - backends/cpu/runtime/** - backends/native/** + - backends/cadence/utils/FACTO + - backends/test/suite/operators/test_linear.py - backends/vulkan/CMakeLists.txt - backends/vulkan/cmake/** - backends/vulkan/runtime/api/** @@ -113,9 +115,10 @@ jobs: if [[ "${NATIVE_COMPONENT}" == "ggml" ]]; then ./install_requirements.sh --use-pt-pinned-commit - # The LLM runner test imports executorch from src/ without installing the - # wheel, so install its export-time dependencies directly. - pip install flatbuffers ruamel.yaml tabulate \ + # The LLM runner test and the op sample generator import executorch from + # src/ without installing the wheel, so install their export-time + # dependencies directly. + pip install flatbuffers gguf ruamel.yaml tabulate \ "$(grep -E '^transformers\b' requirements-examples.txt)" fi diff --git a/backends/native/test/CMakeLists.txt b/backends/native/test/CMakeLists.txt index 523d450ba50..0874426cc68 100644 --- a/backends/native/test/CMakeLists.txt +++ b/backends/native/test/CMakeLists.txt @@ -58,10 +58,10 @@ add_custom_command( ${CMAKE_COMMAND} -E env MKL_CBWR=COMPATIBLE ONEDNN_MAX_CPU_ISA=AVX2 ATEN_CPU_CAPABILITY=avx2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 "PYTHONPATH=${EXECUTORCH_ROOT}/src:${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO" - ${PYTHON_EXECUTABLE} -m executorch.backends.native.test.samples.generate - --out ${_native_op_samples} + FLATC_EXECUTABLE=$ ${PYTHON_EXECUTABLE} -m + executorch.backends.native.test.samples.generate --out ${_native_op_samples} COMMAND ${CMAKE_COMMAND} -E touch ${_native_op_samples}.stamp - DEPENDS ${_native_op_samples_srcs} + DEPENDS flatc ${_native_op_samples_srcs} ${EXECUTORCH_ROOT}/backends/test/suite/operators/test_linear.py VERBATIM ) diff --git a/backends/native/test/runtime/ggml/BUCK b/backends/native/test/runtime/ggml/BUCK index ad23a997116..94105c0a11f 100644 --- a/backends/native/test/runtime/ggml/BUCK +++ b/backends/native/test/runtime/ggml/BUCK @@ -108,8 +108,8 @@ _VARIANTS = [ ] + flags, env = v.get("env", {}) | v.get("sample_env", {}), remote_execution = v.get("remote_execution"), - # TODO: run in CI once they pass. - labels = ci.labels(ci.map(ci.skip_test())), + # TODO: run the others in CI once they pass. + labels = [] if v["suffix"] + name == "_act_quant" else ci.labels(ci.map(ci.skip_test())), test_main = "//executorch/backends/native/test/samples:sample_test_main", deps = [":ggml_engine_host" + v["suffix"]], **v.get("compat", {}) From f67eb1ad942d9cbc741b2701ddf781bcf29df6b1 Mon Sep 17 00:00:00 2001 From: Digant Desai Date: Tue, 3 Nov 2026 06:24:34 -0800 Subject: [PATCH 20/20] [executorch][native] Record dynamic-shape linears as chained bundled samples Pull Request resolved: https://github.com/pytorch/executorch/pull/24405 Engines resize dynamic inputs between inferences, so the op samples now cover a shape changing mid-run. A case built with `chain` is exported with its `Dim`s and recorded as one sample of four calls. Each `Dim` starts mid-range, where export traces it, visits both bounds and returns, every other `Dim` in the opposite order. The harness already runs a sample's calls in order on one executable from one context, resizing inputs between them, so only its header comment changes. The new linear cases are the XNNPACK delegate's dynamic linear tests, each also followed by a ReLU: one and two batch dims, one `Dim` on two dims or two inputs, and sequential and parallel linears, with batch dims down to 1, which tracing mid-range keeps export from specializing. The cases every quantization scheme fits take 256 input features over up to 64 tokens: `dynamic_tokens` in 2-D and, with a static batch of one as in an LLM, in 3-D, two batch dims sharing one `Dim` of up to 16, and the sequential and parallel linears. Authored with Claude Code. Differential Revision: [D128770754](https://our.internmc.facebook.com/intern/diff/D128770754/) **NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D128770754/)! ghstack-source-id: 463657750 --- backends/native/test/samples/cases.py | 32 ++++++- backends/native/test/samples/linear.py | 92 ++++++++++++++++++- backends/native/test/samples/record.py | 33 +++++-- .../native/test/samples/sample_test_main.cpp | 5 +- backends/native/test/samples/test_record.py | 44 ++++++++- 5 files changed, 191 insertions(+), 15 deletions(-) diff --git a/backends/native/test/samples/cases.py b/backends/native/test/samples/cases.py index 82b97316b24..9a613a14db9 100644 --- a/backends/native/test/samples/cases.py +++ b/backends/native/test/samples/cases.py @@ -4,7 +4,8 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -"""Op cases: a model and the inputs of each sample it is recorded with.""" +"""Op cases: a model and the inputs of each sample it is recorded with, or of +the calls of one sample when its shapes are dynamic.""" import inspect from collections.abc import Callable @@ -13,6 +14,7 @@ from typing import TYPE_CHECKING import torch +from torch.export import Dim if TYPE_CHECKING: from facto.inputgen.specs.model import Spec @@ -30,6 +32,9 @@ class Case: model: torch.nn.Module # Positional float32 inputs of each sample; the count may vary by case. inputs: list[Inputs] + # `{dim: Dim}` of each positional input. A dynamic case's inputs are the + # calls of one sample, which run in order on one executable. + dynamic_shapes: tuple[dict[int, Dim], ...] | None = None def outputs(result) -> list[torch.Tensor]: @@ -46,6 +51,31 @@ def draws(count: int, *shapes: tuple[int, ...]) -> list[Inputs]: ] +def chain(name: str, model: torch.nn.Module, *shapes: tuple[int | Dim, ...]) -> Case: + """A dynamic case with standard-normal inputs of `shapes`, from a fixed seed. + Each Dim starts mid-range, which export traces, then visits both bounds and + returns, every other Dim in the opposite order.""" + dims = list(dict.fromkeys(d for s in shapes for d in s if not isinstance(d, int))) + walks = {} + for k, d in enumerate(dims): + mid = (d.min + d.max + 1) // 2 + walks[d] = (mid, d.min, d.max, mid) if k % 2 == 0 else (mid, d.max, d.min, mid) + generator = torch.Generator().manual_seed(0) + inputs = [ + tuple( + torch.randn( + [walks[d][step] if d in walks else d for d in s], generator=generator + ) + for s in shapes + ) + for step in range(4) + ] + dynamic_shapes = tuple( + {i: d for i, d in enumerate(s) if not isinstance(d, int)} for s in shapes + ) + return Case(name, model, inputs, dynamic_shapes) + + class _SuiteRunner: """Stands in for the backend test suite's runner, keeping what a test lowers.""" diff --git a/backends/native/test/samples/linear.py b/backends/native/test/samples/linear.py index 9beea6408fa..6b6a22b1438 100644 --- a/backends/native/test/samples/linear.py +++ b/backends/native/test/samples/linear.py @@ -4,12 +4,13 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -"""Linear cases: the backend test suite's, FACTO's, and shapes and inputs that -stress accumulation and quantization.""" +"""Linear cases: the backend test suite's, FACTO's, shapes and inputs that +stress accumulation and quantization, and dynamic shapes.""" import torch from executorch.backends.native.test.samples.cases import ( Case, + chain, draws, facto_cases, suite_cases, @@ -19,6 +20,7 @@ from executorch.extension.llm.export.gguf import QK_K from facto.inputgen.argument.type import ArgType from facto.inputgen.specs.model import ConstraintProducer as cp, InPosArg, OutArg, Spec +from torch.export import Dim def _unit(*constraints) -> list: @@ -90,6 +92,91 @@ def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, ...]: return tuple(m(x) for m in self.linears) +class _Pair(torch.nn.Module): + """Two modules, each on its own input.""" + + def __init__(self, first: torch.nn.Module, second: torch.nn.Module) -> None: + super().__init__() + self.first, self.second = first, second + + def forward(self, x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, ...]: + return self.first(x), self.second(y) + + +_XNNPACK_SHAPES = ((3, 4, 4), (4, 37, 17), (4, 17, 37)) + + +def _batch_dims(count: int, batch: int) -> tuple[Dim, ...]: + return tuple(Dim(f"batch{i}", min=1, max=batch) for i in range(count)) + + +def _structural( + suffix: str, k: int, hidden: int, n: int, batch: Dim, batch2: Dim +) -> list[Case]: + """Parallel, sequential and both, each linear `k` to `hidden` but the last of + a sequence, `hidden` to `n`.""" + + def sequential() -> torch.nn.Module: + return torch.nn.Sequential( + torch.nn.Linear(k, hidden), torch.nn.Linear(hidden, n) + ) + + return [ + chain( + f"dynamic_parallel{suffix}", + _Pair(torch.nn.Linear(k, hidden), torch.nn.Linear(k, hidden)), + (batch, k), + (batch, k), + ), + chain(f"dynamic_sequential{suffix}", sequential(), (batch, k)), + chain( + f"dynamic_parallel_and_sequential{suffix}", + _Pair(torch.nn.Linear(k, hidden), sequential()), + (batch, k), + (batch2, k), + ), + ] + + +def _dynamic_cases() -> list[Case]: + """The XNNPACK delegate's dynamic linear tests, with batch dims down to 1, + and LLM token shapes every quantization scheme fits.""" + cases = [] + for batch, k, n in _XNNPACK_SHAPES: + for batch_dims in (1, 2): + for bias in (True, False): + name = f"dynamic_{batch_dims}batch_{k}_{n}" + ( + "" if bias else "_no_bias" + ) + shape = (*_batch_dims(batch_dims, batch), k) + cases.append(chain(name, torch.nn.Linear(k, n, bias=bias), shape)) + batch, batch2 = Dim("batch", min=1, max=100), Dim("batch2", min=1, max=100) + cases += [ + chain("dynamic_two_batch", torch.nn.Linear(14, 15), (batch, batch, 14)), + *_structural("", 4, 5, 3, batch, batch2), + ] + tokens = Dim("tokens", min=1, max=64) + cases.append(chain("dynamic_tokens", torch.nn.Linear(256, 64), (tokens, 256))) + for batch, k, n in _XNNPACK_SHAPES: + for batch_dims in (1, 2): + for bias in (True, False): + name = f"dynamic_{batch_dims}batch_{k}_{n}_relu" + ( + "" if bias else "_no_bias" + ) + relu = torch.nn.Sequential( + torch.nn.Linear(k, n, bias=bias), torch.nn.ReLU() + ) + cases.append(chain(name, relu, (*_batch_dims(batch_dims, batch), k))) + rows = Dim("rows", min=1, max=16) + return [ + *cases, + # A static batch of one, as an LLM's prefill and decode. + chain("dynamic_tokens_3d", torch.nn.Linear(256, 64), (1, tokens, 256)), + chain("dynamic_two_batch_256", torch.nn.Linear(256, 64), (rows, rows, 256)), + *_structural("_256", 256, 256, 64, tokens, Dim("tokens2", min=1, max=64)), + ] + + def cases() -> list[Case]: suite = suite_cases(test_linear, samples=4) torch.manual_seed(0) @@ -127,4 +214,5 @@ def cases() -> list[Case]: _facto_case, lambda i, args: i % 5 == 0 or args[1].size(1) % QK_K == 0, ), + *_dynamic_cases(), ] diff --git a/backends/native/test/samples/record.py b/backends/native/test/samples/record.py index 90a57daa399..9465509795e 100644 --- a/backends/native/test/samples/record.py +++ b/backends/native/test/samples/record.py @@ -11,9 +11,10 @@ `precise`, the same math in float64 with quantized weights decoded in float32, whatever the method's dtype, and activations unquantized, floating outputs stored as float32. Weight-only schemes add `golden-int8_act`, the eager outputs -with the same weights and int8 per-token dynamic activations. Nothing in it -depends on an engine or a device: consumers judge an engine by its distance -from `precise` relative to eager's. +with the same weights and int8 per-token dynamic activations. A dynamic case +is exported with its dynamic shapes and recorded as one sample whose calls +change them. Nothing in it depends on an engine or a device: consumers judge an +engine by its distance from `precise` relative to eager's. """ import copy @@ -45,6 +46,13 @@ def _cast(inputs: Inputs, dtype: torch.dtype) -> Inputs: return tuple(t.to(dtype) if t.is_floating_point() else t for t in inputs) +def _samples(calls: list[Call], chained: bool) -> list[Sample]: + """One sample per call, or one running all the calls when they are chained.""" + if chained: + return [Sample(calls)] if calls else [] + return [Sample([call]) for call in calls] + + def record( cases: list[Case], scheme: Scheme, @@ -69,20 +77,27 @@ def record( k: scheme.decode(v).double() for k, v in method.state_dict().items() } inputs = [_cast(x, dtype) for x in case.inputs] - programs[name] = torch.export.export(method, inputs[0]) + chained = case.dynamic_shapes is not None + programs[name] = torch.export.export( + method, + inputs[0], + dynamic_shapes=({"args": case.dynamic_shapes} if chained else None), + ) + golden_calls, precise_calls, act_calls = [], [], [] with torch.no_grad(): for x in inputs: - golden.append(Sample([Call(name, list(x), outputs(method(*x)))])) + golden_calls.append(Call(name, list(x), outputs(method(*x)))) ref = functional_call(method, state, _cast(x, torch.float64)) ref = [ t.float() if t.is_floating_point() else t for t in outputs(ref) ] - precise.append(Sample([Call(name, list(x), ref)])) + precise_calls.append(Call(name, list(x), ref)) if scheme.weight_only: with int8_activations(method): - golden_act.append( - Sample([Call(name, list(x), outputs(method(*x)))]) - ) + act_calls.append(Call(name, list(x), outputs(method(*x)))) + golden += _samples(golden_calls, chained) + precise += _samples(precise_calls, chained) + golden_act += _samples(act_calls, chained) if not programs: logger.info(f"{path.name}: no case fits the scheme, not written") path.unlink(missing_ok=True) diff --git a/backends/native/test/samples/sample_test_main.cpp b/backends/native/test/samples/sample_test_main.cpp index e44d987343c..18e76fde8ec 100644 --- a/backends/native/test/samples/sample_test_main.cpp +++ b/backends/native/test/samples/sample_test_main.cpp @@ -6,8 +6,9 @@ // A gtest main that runs the bundled samples of .ptn packages on the engine the // binary links (see PtnRunner.h): one test per package and method, grouping -// samples by their first call. Each sample runs from a fresh context, and each -// method output is judged over all its samples by OutputCheck. +// samples by their first call. Each sample runs from a fresh context, its calls +// in order on one executable per method, resizing dynamic inputs between them. +// Each method output is judged over all its calls by OutputCheck. // // Usage: // --packages=a.ptn,dir [--rmse_ratio --max_ratio --rtol --sigmas] diff --git a/backends/native/test/samples/test_record.py b/backends/native/test/samples/test_record.py index fd5d28d3b2f..710e89b436c 100644 --- a/backends/native/test/samples/test_record.py +++ b/backends/native/test/samples/test_record.py @@ -12,7 +12,12 @@ import torch import torch.nn.functional as F -from executorch.backends.native.test.samples.cases import Case, draws, suite_cases +from executorch.backends.native.test.samples.cases import ( + Case, + chain, + draws, + suite_cases, +) from executorch.backends.native.test.samples.record import record from executorch.backends.native.test.samples.schemes import ( Gguf, @@ -22,6 +27,7 @@ Torchao, ) from executorch.exir.native import read_samples +from torch.export import Dim from torchao.quantization import IntxWeightOnlyConfig, quantize_ from torchao.quantization.granularity import PerGroup @@ -105,6 +111,42 @@ def test_record_adds_the_8da4w_golden_to_weight_only_packages(self) -> None: got.calls[0].outputs, want.calls[0].outputs, rtol=0, atol=0 ) + def test_chain_visits_each_dims_bounds_starting_mid_range(self) -> None: + a, b = Dim("a", min=1, max=8), Dim("b", min=2, max=4) + case = chain("pair", torch.nn.Identity(), (a, 3), (b, a)) + self.assertEqual( + [[x.shape for x in xs] for xs in case.inputs], + [ + [(5, 3), (3, 5)], + [(1, 3), (4, 1)], + [(8, 3), (2, 8)], + [(5, 3), (3, 5)], + ], + ) + self.assertEqual(case.dynamic_shapes, ({0: a}, {0: b, 1: a})) + + def test_record_chains_a_dynamic_case_into_one_sample(self) -> None: + case = chain("linear", torch.nn.Linear(32, 4), (Dim("batch", min=1, max=6), 32)) + scheme = schemes()["int4wo_g32"] + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "linear.ptn" + record([case], scheme, path, dtypes=["f32"]) + sets = read_samples(str(path)) + self.assertEqual(set(sets), {"golden", "precise", "golden-int8_act"}) + for samples in sets.values(): + (sample,) = samples.samples + self.assertEqual( + [call.inputs[0].shape for call in sample.calls], + [(4, 32), (1, 32), (6, 32), (4, 32)], + ) + model = copy.deepcopy(case.model).eval() + scheme.quantize(model) + with torch.no_grad(): + for call, x in zip( + sets["golden"].samples[0].calls, case.inputs, strict=True + ): + torch.testing.assert_close(call.outputs[0], model(*x), rtol=0, atol=0) + def test_record_keeps_integer_outputs_precise(self) -> None: class Argmax(torch.nn.Module): def forward(self, x: torch.Tensor) -> torch.Tensor: