[executorch][native] Land the bundled op sample harness stack

.ci/scripts/test-native.sh

@@ -36,8 +36,9 @@ case "${component}" in
       -DEXECUTORCH_BUILD_NATIVE_VISION_RUNNER=ON
       -DGGML_NATIVE=OFF
     )
-    targets+=(run_hf_llm_ggml native_ggml_engine_test native_llm_runner_utils_test run_classification_ggml_cpu)
-    test_regex='^native_(ggml_engine|llm_runner_utils)_test$'
+    # TODO: run the other sample tests once they pass.
+    targets+=(run_hf_llm_ggml native_ggml_engine_test native_llm_runner_utils_test run_classification_ggml_cpu native_output_check_test native_ggml_sample_test)
+    test_regex='^native_(ggml_engine|llm_runner_utils|output_check)_test$|^native_ggml_sample_test_act_quant$'
     ;;
   vulkan)
     cmake_args+=(

.github/workflows/test-native.yml

@@ -17,6 +17,8 @@ on:
       - backends/cpu/CMakeLists.txt
       - backends/cpu/runtime/**
       - backends/native/**
+      - backends/cadence/utils/FACTO
+      - backends/test/suite/operators/test_linear.py
       - backends/vulkan/CMakeLists.txt
       - backends/vulkan/cmake/**
       - backends/vulkan/runtime/api/**
@@ -113,9 +115,10 @@ jobs:
 
           if [[ "${NATIVE_COMPONENT}" == "ggml" ]]; then
             ./install_requirements.sh --use-pt-pinned-commit
-            # The LLM runner test imports executorch from src/ without installing the
-            # wheel, so install its export-time dependencies directly.
-            pip install flatbuffers ruamel.yaml tabulate \
+            # The LLM runner test and the op sample generator import executorch from
+            # src/ without installing the wheel, so install their export-time
+            # dependencies directly.
+            pip install flatbuffers gguf ruamel.yaml tabulate \
               "$(grep -E '^transformers\b' requirements-examples.txt)"
           fi
 

backends/native/README.md

@@ -145,6 +145,14 @@ an engine through the interfaces in `runtime/engine/Engine.h`:
 
 An engine rejects a method whose semantics it cannot fully honor.
 
+Engines need not match eager bit for bit: accumulation precision and order,
+fused multiply-add, and the accuracy of functions such as `exp` are up to each
+engine. As a guideline, an engine should round to nearest when it narrows a
+value for storage, with ties to even between floating-point types (f32 to f16 or
+bf16), as eager does. Rounding toward zero, which some GPU APIs allow, biases
+every stored value toward zero, and the bias compounds across layers instead of
+averaging out.
+
 | Engine | Source | Notes |
 | --- | --- | --- |
 | Vulkan | `runtime/vulkan/` | Builds on the ET-VK `ComputeGraph` and shaders. |
@@ -206,7 +214,13 @@ and API. `native_cpu_engine_test` covers the CPU engine when
 `EXECUTORCH_BUILD_NATIVE_CPU` is enabled. `native_vulkan_passes_test` covers the
 Vulkan graph rewrites without requiring a GPU. When either runner below is
 enabled, `native_runner_utils_test` covers their shared runner utilities. Run these
-with `ctest --test-dir cmake-out -R native_`.
+with `ctest --test-dir cmake-out -R native_` after `cmake --build cmake-out`.
+
+The engines' sample tests judge them on the bundled op samples. Building them runs
+the sample generator, which needs `torch`, `torchao`, `gguf` and the FACTO
+submodule at `backends/cadence/utils/FACTO`. Those with failing cases are
+disabled in CTest; run their binaries directly with
+`--packages=<build-dir>/backends/native/test/op_samples`.
 
 The `Test Native Engines` CI workflow has separate Linux jobs for the shared
 runtime, CPU, GGML, and Vulkan, with each test suite run once. To reproduce with
@@ -334,6 +348,9 @@ install/bin/run_hf_llm_ggml --device cpu --cpu_threads 4 \
 The GGML runner accepts `--device cpu|gpu`, `--device_index`, `--cpu_threads`,
 and `--max_cached_graphs` (default 1, 0 rebuilds each call). For GPU execution,
 enable the desired `GGML_*` backend when configuring ExecuTorch, as above.
+GGML quantizes the activations of its quantized matmuls, so it rejects
+weight-only quantized linears, including GGUF k-quants, unless
+`--allow_activation_quant_for_weight_only` accepts that loss of precision.
 
 With `EXECUTORCH_BUILD_TESTS=ON`, `native_llm_runner_utils_test` exercises the
 LLM generation utilities.

backends/native/examples/hf_llm/runtime/run_hf_llm_ggml.cpp

@@ -23,6 +23,11 @@ DEFINE_int32(
     max_cached_graphs,
     1,
     "Graphs kept per executable by input shapes; 0 rebuilds every call.");
+DEFINE_bool(
+    allow_activation_quant_for_weight_only,
+    false,
+    "Run weight-only quantized linears, such as GGUF k-quants, with ggml's "
+    "quantized activations.");
 
 int main(int argc, char** argv) {
   gflags::SetUsageMessage(
@@ -41,6 +46,8 @@ int main(int argc, char** argv) {
       throw std::invalid_argument("--max_cached_graphs must be non-negative");
     }
     options.max_cached_graphs = static_cast<size_t>(FLAGS_max_cached_graphs);
+    options.allow_activation_quant_for_weight_only =
+        FLAGS_allow_activation_quant_for_weight_only;
     return ptn::GgmlEngineHost::create(options);
   });
 }

backends/native/runtime/deserialize/Samples.h

@@ -23,6 +23,8 @@ struct SampleTensor {
   ScalarType dtype = kFloat;
   std::vector<int64_t> sizes;
   std::vector<uint8_t> bytes;
+
+  bool operator==(const SampleTensor&) const = default;
 };
 
 struct Tolerance {

backends/native/runtime/engine/ggml/GgmlEngine.h

@@ -40,6 +40,11 @@ struct GgmlEngineOptions {
   // Prints each newly built graph's node count per ggml op to stderr, which
   // shows whether a lowering rewrite fired.
   bool print_graph_ops = false;
+  // ggml backends quantize the activations of a quantized matmul themselves,
+  // so a linear with quantized weights, int4 or GGUF, whose input is not a
+  // dequantized activation is rejected unless the caller accepts that loss of
+  // precision.
+  bool allow_activation_quant_for_weight_only = false;
 };
 
 class GgmlEngineContext;

backends/native/runtime/engine/ggml/GgmlExecutor.cpp

@@ -542,7 +542,27 @@ class GgmlExecutable final : public EngineExecutable {
             "' may only feed linear or embedding weights, not '" + node.target +
             "'");
       }
+      if (op == "aten.linear.default" &&
+          !options_.allow_activation_quant_for_weight_only &&
+          !has_dequantized_input(node)) {
+        throw std::runtime_error(
+            "ggml: weight-only quantized linear '" + node.name +
+            "' needs allow_activation_quant_for_weight_only, since ggml "
+            "quantizes its activations");
+      }
+    }
+  }
+
+  bool has_dequantized_input(const Node& linear) const {
+    const NamedArgument* input = find_arg(linear, "input", 0);
+    if (input == nullptr || input->arg.kind() != ArgKind::Tensor) {
+      return false;
     }
+    const NodeId producer =
+        method_.graph.value(input->arg.as_tensor().id).producer_id;
+    return valid(producer) &&
+        op_key(method_.graph.node(producer).target) ==
+        "torchao.dequantize_affine.default";
   }
 
   // Converters read some dtypes through typed pointers, so the package's bytes

backends/native/runtime/vulkan/TODO.md

@@ -14,3 +14,23 @@ HuggingFace Llama 3.2 1B 8da4w, 2K export: peak GPU memory is 2794 MiB, against
 2265 MiB with buffer SDPA disabled and 2263 MiB for the ET-VK `.pte`. The
 +528 MiB comes from enabling buffer SDPA. Give these temporaries lifetimes in
 the memory plan, measure again, and attribute whatever remains.
+
+## Lower int4 weight-only linear for any N
+
+`FuseQuantizedLinear` leaves an int4 weight-only linear unlowered when N is not
+a multiple of 4, to match `VK_CHECK_COND(N % 4 == 0)` in ET-VK's
+`prepack_q4_w_4x8_nc_buffer`, and the engine then rejects the quantized weight.
+Round N up to a multiple of 4 there (the pack shader already fills n >= N with
+zeros) and mask the partial last tile on store. Until then Vulkan fails the op
+sample tests `linear_int4wo_*.linear_feature_sizes_2` and
+`linear_int4wo_*.linear_unbatched_0`, where N is 1 and 43.
+
+## Rank-4 linear input
+
+Vulkan returns wrong outputs for the op sample test
+`linear_float.linear_leading_batch_1`, an `nn.Linear(67, 43)` on a
+(2, 4, 8, 67) input: in f32 the fitted rel_bias is -0.48, and f16 returns
+values up to 6.1e4. Both are consistent with only one of the two outer batches
+being written. The rank-3 inputs (4, 8, 67) and (2, 7, 256) pass. Find where
+the outer dimension is lost: the engine's linear lowering, or ET-VK's linear on
+a rank-4 tensor.

backends/native/test/CMakeLists.txt

@@ -29,6 +29,69 @@ target_compile_options(
                               $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-fexceptions>
 )
 
+# Judges an engine's outputs against a package's golden and precise samples.
+add_library(native_output_check STATIC samples/OutputCheck.cpp)
+target_link_libraries(native_output_check PUBLIC native_package)
+add_executable(native_output_check_test samples/test_output_check.cpp)
+target_link_libraries(
+  native_output_check_test PRIVATE native_output_check GTest::gtest_main
+)
+add_test(NAME native_output_check_test COMMAND native_output_check_test)
+foreach(target native_output_check native_output_check_test)
+  target_compile_options(
+    ${target} PRIVATE $<$<CXX_COMPILER_ID:MSVC>:/EHsc>
+                      $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-fexceptions>
+  )
+endforeach()
+
+# Op packages with bundled samples, one per op and quantization scheme. The
+# environment pins the eager kernels' ISA and threads, so the golden is the same
+# on any x86 host with AVX2. Other hosts can differ.
+set(_native_op_samples ${CMAKE_CURRENT_BINARY_DIR}/op_samples)
+file(GLOB_RECURSE _native_op_samples_srcs CONFIGURE_DEPENDS
+     ${CMAKE_CURRENT_SOURCE_DIR}/samples/*.py
+     ${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO/facto/*.py
+)
+add_custom_command(
+  OUTPUT ${_native_op_samples}.stamp
+  COMMAND
+    ${CMAKE_COMMAND} -E env MKL_CBWR=COMPATIBLE ONEDNN_MAX_CPU_ISA=AVX2
+    ATEN_CPU_CAPABILITY=avx2 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1
+    "PYTHONPATH=${EXECUTORCH_ROOT}/src:${EXECUTORCH_ROOT}/backends/cadence/utils/FACTO"
+    FLATC_EXECUTABLE=$<TARGET_FILE:flatc> ${PYTHON_EXECUTABLE} -m
+    executorch.backends.native.test.samples.generate --out ${_native_op_samples}
+  COMMAND ${CMAKE_COMMAND} -E touch ${_native_op_samples}.stamp
+  DEPENDS flatc ${_native_op_samples_srcs}
+          ${EXECUTORCH_ROOT}/backends/test/suite/operators/test_linear.py
+  VERBATIM
+)
+add_custom_target(native_op_samples DEPENDS ${_native_op_samples}.stamp)
+
+if(EXECUTORCH_BUILD_NATIVE_GGML OR EXECUTORCH_BUILD_NATIVE_VULKAN)
+  if(NOT TARGET gflags AND NOT TARGET gflags::gflags)
+    find_package(gflags REQUIRED)
+  endif()
+  if(TARGET gflags::gflags)
+    set(_native_test_gflags gflags::gflags)
+  else()
+    set(_native_test_gflags gflags)
+  endif()
+
+  # A gtest main judging the engine whose host a test links on the bundled
+  # samples of the packages it names; engine tests set its flags.
+  add_library(native_sample_test_main OBJECT samples/sample_test_main.cpp)
+  target_link_libraries(
+    native_sample_test_main
+    PUBLIC native_output_check native_runtime native_package native_engine
+           ${_native_test_gflags} GTest::gtest
+  )
+  target_compile_options(
+    native_sample_test_main
+    PRIVATE $<$<CXX_COMPILER_ID:MSVC>:/EHsc>
+            $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-fexceptions>
+  )
+endif()
+
 if(EXECUTORCH_BUILD_NATIVE_VULKAN)
   add_executable(
     native_vulkan_passes_test
@@ -50,6 +113,23 @@ if(EXECUTORCH_BUILD_NATIVE_VULKAN)
     PRIVATE $<$<CXX_COMPILER_ID:MSVC>:/EHsc>
             $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-fexceptions>
   )
+
+  add_executable(native_vulkan_sample_test runtime/vulkan/vulkan_ptn_runner.cpp)
+  target_link_libraries(
+    native_vulkan_sample_test PRIVATE native_sample_test_main
+                                      native_vulkan_engine
+  )
+  add_dependencies(native_vulkan_sample_test native_op_samples)
+  add_test(NAME native_vulkan_sample_test
+           COMMAND native_vulkan_sample_test --packages=${_native_op_samples}
+  )
+  # TODO: Enable once every case passes.
+  set_tests_properties(native_vulkan_sample_test PROPERTIES DISABLED TRUE)
+  target_compile_options(
+    native_vulkan_sample_test
+    PRIVATE $<$<CXX_COMPILER_ID:MSVC>:/EHsc>
+            $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-fexceptions>
+  )
 endif()
 
 if(EXECUTORCH_BUILD_NATIVE_GGML)
@@ -65,6 +145,29 @@ if(EXECUTORCH_BUILD_NATIVE_GGML)
     PRIVATE $<$<CXX_COMPILER_ID:MSVC>:/EHsc>
             $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-fexceptions>
   )
+
+  add_executable(native_ggml_sample_test runtime/ggml/ggml_ptn_runner.cpp)
+  target_link_libraries(
+    native_ggml_sample_test PRIVATE native_sample_test_main native_ggml_engine
+  )
+  add_dependencies(native_ggml_sample_test native_op_samples)
+  add_test(NAME native_ggml_sample_test
+           COMMAND native_ggml_sample_test --packages=${_native_op_samples}
+                   --device=cpu
+  )
+  # TODO: Enable once every case passes.
+  set_tests_properties(native_ggml_sample_test PROPERTIES DISABLED TRUE)
+  # native_ggml_sample_test_act_quant lets ggml quantize the activations of
+  # weight-only quantized linears, judged against golden-int8_act.
+  add_test(NAME native_ggml_sample_test_act_quant
+           COMMAND native_ggml_sample_test --packages=${_native_op_samples}
+                   --device=cpu --allow_activation_quant_for_weight_only
+  )
+  target_compile_options(
+    native_ggml_sample_test
+    PRIVATE $<$<CXX_COMPILER_ID:MSVC>:/EHsc>
+            $<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-fexceptions>
+  )
 endif()
 
 if(TARGET native_runner_utils)

backends/native/test/e2e/test_cross_engine.py

@@ -35,7 +35,14 @@
 _MAX_SEQ = 16
 
 
-def _run(runner: str, package: Path, steps, outdir: Path, resize: bool = False):
+def _run(
+    runner: str,
+    package: Path,
+    steps,
+    outdir: Path,
+    resize: bool = False,
+    flags: tuple[str, ...] = (),
+):
     """Runs each step's inputs in order on one executable; returns each step's
     single float32 output."""
     input_steps, output_steps, shapes = [], [], []
@@ -55,6 +62,7 @@ def _run(runner: str, package: Path, steps, outdir: Path, resize: bool = False):
     ]
     if resize:
         command.append(f"--input_sizes={';'.join(shapes)}")
+    command.extend(flags)
     resources = files(__package__)
     with as_file(resources / runner) as binary, as_file(
         resources / "vulkan_driver"
@@ -390,11 +398,13 @@ def test_inplace_relu_of_dynamic_view(self):
         seq = Dim("seq", min=2, max=12)
         self._check_dynamic(_ViewRelu(), steps, ({1: seq},))
 
-    def _check_engines(self, package, steps, expected, outdir, atol_fraction, engines):
+    def _check_engines(
+        self, package, steps, expected, outdir, atol_fraction, engines, flags=()
+    ):
         tokens = {}
         for runner in engines:
             with self.subTest(engine=runner):
-                outputs = _run(runner, package, steps, outdir, resize=True)
+                outputs = _run(runner, package, steps, outdir, resize=True, flags=flags)
                 self.assertEqual(len(outputs), len(expected))
                 for step, (actual, reference) in enumerate(zip(outputs, expected)):
                     self.assertEqual(actual.shape, reference.shape)
@@ -433,6 +443,7 @@ def _check_llama(
         atol_fraction: float,
         engines=_ENGINES,
         max_seq: int = _MAX_SEQ,
+        flags: tuple[str, ...] = (),
     ):
         prompt = torch.randint(0, model.config.vocab_size, (1, 7))
         eager = LastTokenLM(copy.deepcopy(model), max_seq, kv_cache_dtype).eval()
@@ -457,7 +468,7 @@ def _check_llama(
             package = outdir / "tiny_llama.ptn"
             to_native(ep).save(str(package))
             tokens = self._check_engines(
-                package, steps, expected, outdir, atol_fraction, engines
+                package, steps, expected, outdir, atol_fraction, engines, flags
             )
         greedy = [e[0, -1].argmax().item() for e in expected]
         for runner, chosen in tokens.items():
@@ -472,16 +483,20 @@ def test_tiny_llama_gguf(self):
         model = _tiny_llama()
         pairs = _gguf_weights(model)
         load_gguf(model, "tiny.gguf", "llama", lambda _: iter(pairs))
-        # ggml requantizes activations to Q8_K for its K-quant matmuls, while
-        # native-vk decodes the K-quant weights exactly to fp32. Re-seeding
-        # gives both engines the same prompt.
-        for runner, atol_fraction in (
-            ("ggml_ptn_runner", 5e-2),
-            ("vulkan_ptn_runner", 1e-3),
+        # ggml requantizes activations to Q8_K for its K-quant matmuls, once
+        # allowed to, while native-vk decodes the K-quant weights exactly to
+        # fp32. Re-seeding gives both engines the same prompt.
+        for runner, atol_fraction, flags in (
+            ("ggml_ptn_runner", 5e-2, ("--allow_activation_quant_for_weight_only",)),
+            ("vulkan_ptn_runner", 1e-3, ()),
         ):
             torch.manual_seed(0)
             self._check_llama(
-                copy.deepcopy(model), torch.float32, atol_fraction, engines=(runner,)
+                copy.deepcopy(model),
+                torch.float32,
+                atol_fraction,
+                engines=(runner,),
+                flags=flags,
             )
 
     def test_tiny_llama_fp16_kv(self):