[executorch][native] Judge activation-quantized engines against an int8-activation golden

.ci/scripts/test-native.sh

@@ -36,8 +36,9 @@ case "${component}" in
       -DEXECUTORCH_BUILD_NATIVE_VISION_RUNNER=ON
       -DGGML_NATIVE=OFF
     )
-    targets+=(run_hf_llm_ggml native_ggml_engine_test native_llm_runner_utils_test run_classification_ggml_cpu)
-    test_regex='^native_(ggml_engine|llm_runner_utils)_test$'
+    # TODO: run the other sample tests once they pass.
+    targets+=(run_hf_llm_ggml native_ggml_engine_test native_llm_runner_utils_test run_classification_ggml_cpu native_output_check_test native_ggml_sample_test)
+    test_regex='^native_(ggml_engine|llm_runner_utils|output_check)_test$|^native_ggml_sample_test_act_quant$'
     ;;
   vulkan)
     cmake_args+=(

.github/workflows/test-native.yml

@@ -17,6 +17,8 @@ on:
       - backends/cpu/CMakeLists.txt
       - backends/cpu/runtime/**
       - backends/native/**
+      - backends/cadence/utils/FACTO
+      - backends/test/suite/operators/test_linear.py
       - backends/vulkan/CMakeLists.txt
       - backends/vulkan/cmake/**
       - backends/vulkan/runtime/api/**
@@ -113,9 +115,10 @@ jobs:
 
           if [[ "${NATIVE_COMPONENT}" == "ggml" ]]; then
             ./install_requirements.sh --use-pt-pinned-commit
-            # The LLM runner test imports executorch from src/ without installing the
-            # wheel, so install its export-time dependencies directly.
-            pip install flatbuffers ruamel.yaml tabulate \
+            # The LLM runner test and the op sample generator import executorch from
+            # src/ without installing the wheel, so install their export-time
+            # dependencies directly.
+            pip install flatbuffers gguf ruamel.yaml tabulate \
               "$(grep -E '^transformers\b' requirements-examples.txt)"
           fi
 

backends/native/README.md

@@ -145,6 +145,14 @@ an engine through the interfaces in `runtime/engine/Engine.h`:
 
 An engine rejects a method whose semantics it cannot fully honor.
 
+Engines need not match eager bit for bit: accumulation precision and order,
+fused multiply-add, and the accuracy of functions such as `exp` are up to each
+engine. As a guideline, an engine should round to nearest when it narrows a
+value for storage, with ties to even between floating-point types (f32 to f16 or
+bf16), as eager does. Rounding toward zero, which some GPU APIs allow, biases
+every stored value toward zero, and the bias compounds across layers instead of
+averaging out.
+
 | Engine | Source | Notes |
 | --- | --- | --- |
 | Vulkan | `runtime/vulkan/` | Builds on the ET-VK `ComputeGraph` and shaders. |
@@ -209,8 +217,9 @@ enabled, `native_runner_utils_test` covers their shared runner utilities. Run th
 with `ctest --test-dir cmake-out -R native_` after `cmake --build cmake-out`.
 
 The engines' sample tests judge them on the bundled op samples. Building them runs
-the sample generator, which needs `torch` and `torchao`. Those with failing cases
-are disabled in CTest; run their binaries directly with
+the sample generator, which needs `torch`, `torchao`, `gguf` and the FACTO
+submodule at `backends/cadence/utils/FACTO`. Those with failing cases are
+disabled in CTest; run their binaries directly with
 `--packages=<build-dir>/backends/native/test/op_samples`.
 
 The `Test Native Engines` CI workflow has separate Linux jobs for the shared

backends/native/runtime/vulkan/TODO.md

@@ -14,3 +14,23 @@ HuggingFace Llama 3.2 1B 8da4w, 2K export: peak GPU memory is 2794 MiB, against
 2265 MiB with buffer SDPA disabled and 2263 MiB for the ET-VK `.pte`. The
 +528 MiB comes from enabling buffer SDPA. Give these temporaries lifetimes in
 the memory plan, measure again, and attribute whatever remains.
+
+## Lower int4 weight-only linear for any N
+
+`FuseQuantizedLinear` leaves an int4 weight-only linear unlowered when N is not
+a multiple of 4, to match `VK_CHECK_COND(N % 4 == 0)` in ET-VK's
+`prepack_q4_w_4x8_nc_buffer`, and the engine then rejects the quantized weight.
+Round N up to a multiple of 4 there (the pack shader already fills n >= N with
+zeros) and mask the partial last tile on store. Until then Vulkan fails the op
+sample tests `linear_int4wo_*.linear_feature_sizes_2` and
+`linear_int4wo_*.linear_unbatched_0`, where N is 1 and 43.
+
+## Rank-4 linear input
+
+Vulkan returns wrong outputs for the op sample test
+`linear_float.linear_leading_batch_1`, an `nn.Linear(67, 43)` on a
+(2, 4, 8, 67) input: in f32 the fitted rel_bias is -0.48, and f16 returns
+values up to 6.1e4. Both are consistent with only one of the two outer batches
+being written. The rank-3 inputs (4, 8, 67) and (2, 7, 256) pass. Find where
+the outer dimension is lost: the engine's linear lowering, or ET-VK's linear on
+a rank-4 tensor.

backends/native/test/CMakeLists.txt

@@ -48,18 +48,20 @@ endforeach()
 # environment pins the eager kernels' ISA and threads, so the golden is the same
 # on any x86 host with AVX2. Other hosts can differ.
 set(_native_op_samples ${CMAKE_CURRENT_BINARY_DIR}/op_samples)
-file(GLOB _native_op_samples_srcs CONFIGURE_DEPENDS
+file(GLOB_RECURSE _native_op_samples_srcs CONFIGURE_DEPENDS
      ${CMAKE_CURRENT_SOURCE_DIR}/samples/*.py
+     ${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO/facto/*.py
 )
 add_custom_command(
   OUTPUT ${_native_op_samples}.stamp
   COMMAND
     ${CMAKE_COMMAND} -E env MKL_CBWR=COMPATIBLE ONEDNN_MAX_CPU_ISA=AVX2
     ATEN_CPU_CAPABILITY=avx2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1
-    "PYTHONPATH=${EXECUTORCH_ROOT}/src" ${PYTHON_EXECUTABLE} -m
+    "PYTHONPATH=${EXECUTORCH_ROOT}/src:${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO"
+    FLATC_EXECUTABLE=$<TARGET_FILE:flatc> ${PYTHON_EXECUTABLE} -m
     executorch.backends.native.test.samples.generate --out ${_native_op_samples}
   COMMAND ${CMAKE_COMMAND} -E touch ${_native_op_samples}.stamp
-  DEPENDS ${_native_op_samples_srcs}
+  DEPENDS flatc ${_native_op_samples_srcs}
           ${EXECUTORCH_ROOT}/backends/test/suite/operators/test_linear.py
   VERBATIM
 )
@@ -155,6 +157,12 @@ if(EXECUTORCH_BUILD_NATIVE_GGML)
   )
   # TODO: Enable once every case passes.
   set_tests_properties(native_ggml_sample_test PROPERTIES DISABLED TRUE)
+  # native_ggml_sample_test_act_quant lets ggml quantize the activations of
+  # weight-only quantized linears, judged against golden-int8_act.
+  add_test(NAME native_ggml_sample_test_act_quant
+           COMMAND native_ggml_sample_test --packages=${_native_op_samples}
+                   --device=cpu --allow_activation_quant_for_weight_only
+  )
   target_compile_options(
     native_ggml_sample_test
     PRIVATE $<$<CXX_COMPILER_ID:MSVC>:/EHsc>

backends/native/test/runtime/PtnRunner.h

@@ -12,8 +12,15 @@
 
 namespace ptn::testing {
 
+// Numerics a driver lets an engine trade away. Drivers then judge the engine as
+// having made the trade, so an engine that never makes it rejects the option.
+struct HostOptions {
+  // Quantize the activations of linears whose weights alone are quantized.
+  bool allow_activation_quant_for_weight_only = false;
+};
+
 // Defined once per binary by an engine's host library; drivers such as
 // ptn_runner_main.cpp run every engine through the host it returns.
-std::unique_ptr<EngineHost> make_engine_host();
+std::unique_ptr<EngineHost> make_engine_host(const HostOptions& options);
 
 } // namespace ptn::testing

backends/native/test/runtime/ggml/BUCK

@@ -45,6 +45,11 @@ _VARIANTS = [
             # Buck does not put the CUDA driver stub's library on the loader path.
             # @oss-disable[end= ]: "LD_PRELOAD": "$(location //tools/lazify:lazy_stub_binder[shared])",
         },
+        # ggml-CUDA computes f16 BLAS GEMMs in f16 and f32 ones in TF32 by default.
+        "sample_env": {
+            "GGML_CUDA_CUBLAS_COMPUTE_TYPE": "f32",
+            "NVIDIA_TF32_OVERRIDE": "0",
+        },
     },
     {
         "suffix": "_hip",
@@ -91,23 +96,29 @@ _VARIANTS = [
     for v in _VARIANTS
 ]
 
+# ggml_sample_test_act_quant lets ggml quantize the activations of weight-only
+# quantized linears, judged against golden-int8_act.
 [
     fbcode_target(
         _kind = runtime.cxx_test,
-        name = "ggml_sample_test" + v["suffix"],
+        name = "ggml_sample_test" + name + v["suffix"],
         args = [
             "--packages=$(location //executorch/backends/native/test/samples:op_samples)",
             "--device=" + v["device"],
-        ],
-        env = v.get("env", {}),
+        ] + flags,
+        env = v.get("env", {}) | v.get("sample_env", {}),
         remote_execution = v.get("remote_execution"),
-        # TODO: run in CI once they pass.
-        labels = ci.labels(ci.map(ci.skip_test())),
+        # TODO: run the others in CI once they pass.
+        labels = [] if v["suffix"] + name == "_act_quant" else ci.labels(ci.map(ci.skip_test())),
         test_main = "//executorch/backends/native/test/samples:sample_test_main",
         deps = [":ggml_engine_host" + v["suffix"]],
         **v.get("compat", {})
     )
     for v in _VARIANTS
+    for name, flags in [
+        ("", []),
+        ("_act_quant", ["--allow_activation_quant_for_weight_only"]),
+    ]
 ]
 
 [

backends/native/test/runtime/ggml/ggml_ptn_runner.cpp

@@ -26,15 +26,11 @@ DEFINE_bool(
     print_graph_ops,
     false,
     "Print each built graph's node count per ggml op to stderr.");
-DEFINE_bool(
-    allow_activation_quant_for_weight_only,
-    false,
-    "Run weight-only quantized linears with ggml's quantized activations.");
 
 namespace ptn::testing {
 
 // cppcheck-suppress unusedFunction
-std::unique_ptr<EngineHost> make_engine_host() {
+std::unique_ptr<EngineHost> make_engine_host(const HostOptions& host_options) {
   GgmlEngineOptions options;
   if (FLAGS_device == "gpu") {
     options.device = GgmlDevice::Gpu;
@@ -49,7 +45,7 @@ std::unique_ptr<EngineHost> make_engine_host() {
   options.max_cached_graphs = static_cast<size_t>(FLAGS_max_cached_graphs);
   options.print_graph_ops = FLAGS_print_graph_ops;
   options.allow_activation_quant_for_weight_only =
-      FLAGS_allow_activation_quant_for_weight_only;
+      host_options.allow_activation_quant_for_weight_only;
   return GgmlEngineHost::create(options);
 }
 

backends/native/test/runtime/ptn_runner_main.cpp

@@ -65,6 +65,10 @@ DEFINE_int32(
     iterations,
     1,
     "Timed executions; above one, prints median and min latency.");
+DEFINE_bool(
+    allow_activation_quant_for_weight_only,
+    false,
+    "Let the engine quantize the activations of weight-only quantized linears.");
 
 namespace {
 
@@ -201,7 +205,9 @@ void run() {
   const ptn::ByteSpan program_bytes = package->program_bytes();
   const auto program = std::make_shared<const ptn::Program>(
       ptn::Program::load(program_bytes.data(), program_bytes.size()));
-  const auto host = ptn::testing::make_engine_host();
+  const auto host = ptn::testing::make_engine_host(
+      {.allow_activation_quant_for_weight_only =
+           FLAGS_allow_activation_quant_for_weight_only});
   const auto context = host->create_context(program, package);
   const auto executable = context->compile(FLAGS_method);
 

backends/native/test/runtime/vulkan/vulkan_ptn_runner.cpp

@@ -6,13 +6,20 @@
 
 // The Vulkan engine host for PTN test drivers.
 
+#include <stdexcept>
+
 #include <executorch/backends/native/runtime/vulkan/VulkanEngine.h>
 #include <executorch/backends/native/test/runtime/PtnRunner.h>
 
 namespace ptn::testing {
 
 // cppcheck-suppress unusedFunction
-std::unique_ptr<EngineHost> make_engine_host() {
+std::unique_ptr<EngineHost> make_engine_host(const HostOptions& options) {
+  if (options.allow_activation_quant_for_weight_only) {
+    throw std::invalid_argument(
+        "vulkan: never quantizes activations, so it rejects "
+        "allow_activation_quant_for_weight_only");
+  }
   return VulkanEngineHost::create();
 }
 

backends/native/test/samples/BUCK

@@ -17,7 +17,11 @@ fbcode_target(
         "//caffe2:torch",
         "//executorch/backends/test/suite/operators:test_operators",
         "//executorch/exir/native:native",
+        "//executorch/extension/llm/export:gguf",
         "//pytorch/ao:torchao",
+        "//pytorch/facto:facto",
+        "fbsource//third-party/pypi/gguf:gguf",
+        "fbsource//third-party/pypi/numpy:numpy",
     ],
 )
 
@@ -37,6 +41,7 @@ fbcode_target(
         ":record",
         "//caffe2:torch",
         "//executorch/exir/native:native",
+        "//pytorch/ao:torchao",
     ],
 )
 

backends/native/test/samples/cases.py

@@ -7,11 +7,16 @@
 """Op cases: a model and the inputs of each sample it is recorded with."""
 
 import inspect
+from collections.abc import Callable
 from dataclasses import dataclass
 from types import ModuleType
+from typing import TYPE_CHECKING
 
 import torch
 
+if TYPE_CHECKING:
+    from facto.inputgen.specs.model import Spec
+
 Inputs = tuple[torch.Tensor, ...]
 
 # Caps a suite case's input and output elements over its samples, keeping at
@@ -85,3 +90,20 @@ def suite_cases(module: ModuleType, samples: int) -> list[Case]:
     if not cases:
         raise ValueError(f"no test in {module.__name__} lowers a model")
     return cases
+
+
+def facto_cases(
+    spec: "Spec", build: Callable[..., Case], keep: Callable[[int, tuple], bool]
+) -> list[Case]:
+    """One-sample cases `build(name, *args)` from FACTO's argument tuples for
+    `spec`, named by position, keeping those where `keep(position, args)`."""
+    # FACTO is on the path for the sample generator only, not for pytest.
+    from facto.inputgen.argtuple.gen import ArgumentTupleGenerator
+    from facto.inputgen.utils.random_manager import seeded_random_manager
+
+    seeded_random_manager.seed(0)
+    return [
+        build(f"facto_{i}", *args)
+        for i, (args, _, _) in enumerate(ArgumentTupleGenerator(spec).gen())
+        if keep(i, args)
+    ]