feat(provenance): resolve source git SHA via a validated multi-channel chain

.dockerignore

@@ -0,0 +1,6 @@
+# Generated build-time provenance (scripts/write_build_info.py). A locally-run
+# generator leaves this in the source tree; excluding it from the Docker build
+# context stops a stale baked SHA from being COPYd into the image and silently
+# overriding the ENDPOINTS_GIT_SHA build-arg (resolve_git_sha prefers baked).
+src/inference_endpoint/_build_info.py
+src/inference_endpoint/_build_info.py.tmp

.gitignore

@@ -212,3 +212,9 @@ CLAUDE.local.md
 
 # Generated dataset cache (created by Dataset.get_dataloader())
 dataset_cache/
+
+# Generated build-time provenance (scripts/write_build_info.py). Packed into
+# wheels by uv_build; NOT into images (.dockerignored — images carry the SHA via
+# the ENDPOINTS_GIT_SHA build-arg). The .tmp is the atomic-write staging file.
+src/inference_endpoint/_build_info.py
+src/inference_endpoint/_build_info.py.tmp

docs/metrics/DESIGN.md

@@ -76,6 +76,7 @@ subscriber's last live snapshot and the resulting `Report.complete` is `False`.
 class Report(msgspec.Struct, frozen=True):
     version: str
     git_sha: str | None
+    git_sha_source: str  # which channel resolved git_sha: baked | env | git | none
     test_started_at: int
     n_samples_issued: int
     n_samples_completed: int

docs/metrics/report_design.md

@@ -66,7 +66,7 @@ p50. A zero-count series returns `{}` (or an all-null early-stopping map if the
 
 ### `Report` (frozen `msgspec.Struct`)
 
-Fields: `version`, `git_sha`, `test_started_at`, `n_samples_issued/completed/failed`,
+Fields: `version`, `git_sha`, `git_sha_source`, `test_started_at`, `n_samples_issued/completed/failed`,
 `duration_ns`, `state`, `complete`, the five rollup dicts (`ttft`, `tpot`, `latency`,
 `input_sequence_lengths`, `output_sequence_lengths`), `legacy_loadgen_window_duration_ns`,
 `qps`, `tps`,

docs/utils/DESIGN.md

@@ -27,8 +27,9 @@ does import from other `inference_endpoint` subpackages.
 **No cross-imports from `utils/` helper modules**
 
 `logging.py` and `dataset_utils.py` stay lightweight and broadly reusable. `version.py` is also
-small, but it intentionally imports `inference_endpoint.__version__` and shells out to `git` to
-report build metadata. `benchmark_httpclient.py` is exempt entirely: it is a standalone tool, not
+small, but it intentionally imports `inference_endpoint.__version__` and resolves a build SHA
+through a fallback chain — a baked `_build_info.py`, the `ENDPOINTS_GIT_SHA` env var, then a live
+`git` query — to report build metadata. `benchmark_httpclient.py` is exempt entirely: it is a standalone tool, not
 a reusable helper.
 
 **`benchmark_httpclient.py` is a standalone tool**
@@ -44,7 +45,8 @@ uv run python -m inference_endpoint.utils.benchmark_httpclient --endpoint URL --
 
 ## Integration Points
 
-| Consumer           | Usage                                        |
-| ------------------ | -------------------------------------------- |
-| `main.py`          | Calls `setup_logging()` at startup           |
-| `commands/info.py` | Imports `__version__` for the `info` command |
+| Consumer            | Usage                                                                                                  |
+| ------------------- | ------------------------------------------------------------------------------------------------------ |
+| `main.py`           | Calls `setup_logging()` at startup                                                                     |
+| `commands/info.py`  | Imports `__version__` for the `info` command                                                           |
+| `metrics/report.py` | `Report.from_snapshot` calls `get_version_info()` to record `git_sha` / `git_sha_source` in the report |

scripts/Dockerfile.dev

@@ -1,6 +1,7 @@
 # Development Dockerfile for the MLPerf Inference Endpoint Benchmarking System
-# From project root:
-# docker build -f scripts/Dockerfile.dev --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev .
+# From project root (pass GIT_SHA so the client reports its source commit; see the
+# provenance note at the bottom of this file):
+# docker build -f scripts/Dockerfile.dev --build-arg GIT_SHA=$(git rev-parse --short=7 HEAD) --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t inference-endpoint-dev .
 # docker run -v $(pwd):/mnt/inference-endpoint -it --shm-size=512m inference-endpoint-dev bash
 #
 # VBench (WAN 2.2) accuracy scorer is ON by default (consistent with the DeepSeek-R1
@@ -102,3 +103,17 @@ RUN if [ "${PROVISION_VBENCH}" = "1" ]; then \
     else \
         echo "PROVISION_VBENCH=${PROVISION_VBENCH}: skipping VBench provisioning" ; \
     fi
+
+# Runtime provenance: expose the source git SHA so `inference-endpoint` prints it
+# in its report. This image COPYs src/ but not .git, so live `git rev-parse` finds
+# no repo; resolve_git_sha() reads ENDPOINTS_GIT_SHA instead. Kept LAST so the
+# per-commit SHA (changes every build) doesn't invalidate the cached dependency
+# and provisioning layers above. Pass it at build time:
+#   docker build ... --build-arg GIT_SHA=$(git rev-parse --short=7 HEAD)
+# Default is empty (NOT "unknown"): resolve_git_sha validates the value, so an
+# empty/omitted arg falls through to source=none rather than recording a bogus
+# "unknown" as an env attestation.
+# Image-level provenance (the OCI image.revision annotation / :<sha> tag) is set
+# separately by the publish script on the pushed manifest, so no LABEL here.
+ARG GIT_SHA=
+ENV ENDPOINTS_GIT_SHA=${GIT_SHA}

scripts/write_build_info.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Bake the build-time git SHA into ``src/inference_endpoint/_build_info.py``.
+
+Run from a source checkout (where ``.git`` exists) BEFORE building a wheel so the
+SHA travels with the artifact and is readable at runtime without git::
+
+    python scripts/write_build_info.py
+    uv build
+
+The generated file is gitignored but uv_build still packs it into the wheel (it
+selects by module tree, not git tracking). ``utils.version.resolve_git_sha``
+imports it as the highest-priority provenance source; without it, runtime falls
+back to the ``ENDPOINTS_GIT_SHA`` env var (containers) then a live ``git`` query
+(dev). Container images do NOT use this file (it is ``.dockerignore``d); they
+carry the SHA via the ``ENDPOINTS_GIT_SHA`` build-arg instead.
+
+The SHA format mirrors ``get_git_sha``: the 7-char short SHA plus a ``-dirty``
+suffix when tracked files have uncommitted changes. Stdlib-only on purpose — this
+runs before ``inference_endpoint`` is installed, so it cannot import from it.
+"""
+
+import os
+import re
+import subprocess
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parents[1]
+_OUTPUT = _REPO_ROOT / "src" / "inference_endpoint" / "_build_info.py"
+
+# Build-time git budget: more lenient than runtime's snappy 1.0s (the build can
+# afford to wait); intentionally not shared with utils.version (stdlib-only here).
+_GIT_TIMEOUT_S = 5.0
+# Bare hex object name (SHA-1 short/full or SHA-256). This is the hex CORE of
+# utils.version._SHA_RE, which additionally allows a trailing "-dirty"; here the
+# SHA is validated before write_build_info appends "-dirty" itself.
+_SHA_RE = re.compile(r"[0-9a-fA-F]{7,64}")
+
+_LICENSE_HEADER = """\
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+
+
+# Git's repo-location env family. A CI wrapper or ``git submodule foreach`` can
+# export any of these to point git at a foreign repo/index; dropping the whole
+# family (plus the toplevel check below) stops the generator from baking a SHA or
+# dirty bit that isn't this source tree's. Mirrors utils.version's copy (that
+# module can't be imported here — this script runs pre-install, stdlib-only).
+_GIT_LOCATION_ENV_VARS = (
+    "GIT_DIR",
+    "GIT_WORK_TREE",
+    "GIT_INDEX_FILE",
+    "GIT_COMMON_DIR",
+    "GIT_OBJECT_DIRECTORY",
+    "GIT_ALTERNATE_OBJECT_DIRECTORIES",
+)
+
+
+def _git_env() -> dict[str, str]:
+    """Environment with the git repo-location family dropped (see the note above)."""
+    env = dict(os.environ)
+    for var in _GIT_LOCATION_ENV_VARS:
+        env.pop(var, None)
+    return env
+
+
+def _short_sha(repo_root: Path) -> str:
+    """Return the validated short SHA of the repo rooted at ``repo_root``.
+
+    Verifies the discovered toplevel IS ``repo_root`` (mirrors get_git_sha's
+    foreign-repo guard) and that the value is hex, so the generator cannot bake a
+    foreign or malformed SHA that runtime would otherwise trust as baked ground
+    truth. Raises on any mismatch so a bad bake fails loudly instead of writing
+    a value runtime silently discards.
+    """
+    result = subprocess.run(
+        ["git", "rev-parse", "--show-toplevel", "--short=7", "HEAD"],
+        cwd=repo_root,
+        env=_git_env(),
+        capture_output=True,
+        text=True,
+        timeout=_GIT_TIMEOUT_S,
+        check=True,
+    )
+    lines = result.stdout.splitlines()
+    if len(lines) != 2:
+        raise RuntimeError(f"unexpected git rev-parse output: {result.stdout!r}")
+    toplevel, sha = lines
+    if Path(toplevel).resolve() != repo_root.resolve():
+        raise RuntimeError(
+            f"refusing to bake a SHA from a different repo: git toplevel "
+            f"{toplevel!r} != {repo_root}"
+        )
+    sha = sha.strip()
+    if not _SHA_RE.fullmatch(sha):
+        raise RuntimeError(f"git returned a non-hex SHA: {sha!r}")
+    return sha
+
+
+def _tree_dirty(repo_root: Path) -> bool:
+    """True iff tracked files have uncommitted changes; fails to dirty.
+
+    Mirrors ``utils.version._git_tree_dirty``: untracked files are ignored, and
+    any outcome other than a clean exit 0 (non-zero code, timeout, OSError) is
+    reported as dirty so a build never bakes a clean SHA it could not verify.
+
+    Caveat (matches ``git describe --dirty`` semantics): a NEW untracked file
+    under the packaged tree (``src/inference_endpoint/``) is NOT flagged, yet
+    uv_build would pack it — so baking + building a new module without committing
+    it yields a clean SHA for a wheel whose code differs from that commit. Commit
+    new modules before baking.
+    """
+    try:
+        result = subprocess.run(
+            ["git", "diff-index", "--quiet", "HEAD", "--"],
+            cwd=repo_root,
+            env=_git_env(),
+            capture_output=True,
+            timeout=_GIT_TIMEOUT_S,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return True
+    return result.returncode != 0
+
+
+def write_build_info(repo_root: Path = _REPO_ROOT, output: Path = _OUTPUT) -> str:
+    """Generate ``output`` with the (validated) resolved SHA and return that SHA."""
+    sha = _short_sha(repo_root)
+    if _tree_dirty(repo_root):
+        sha = f"{sha}-dirty"
+    content = (
+        f"{_LICENSE_HEADER}\n"
+        '"""Generated by scripts/write_build_info.py at build time. Do not edit or commit."""\n'
+        "\n"
+        # _short_sha validated the SHA is hex; !r additionally guarantees the
+        # generated module parses regardless of the value.
+        f"GIT_SHA = {sha!r}\n"
+    )
+    # Atomic: write a sibling temp file then rename, so a crash mid-write cannot
+    # leave a truncated _build_info.py that would SyntaxError on import. The
+    # finally cleans up the temp on any failure before the rename.
+    tmp = output.with_name(output.name + ".tmp")
+    try:
+        tmp.write_text(content)
+        tmp.replace(output)
+    finally:
+        tmp.unlink(missing_ok=True)
+    return sha
+
+
+def main() -> None:
+    sha = write_build_info()
+    print(f"Wrote {_OUTPUT} (GIT_SHA={sha})")
+
+
+if __name__ == "__main__":
+    main()

src/inference_endpoint/metrics/report.py

@@ -181,6 +181,11 @@ class Report(msgspec.Struct, frozen=True):  # type: ignore[call-arg]
 
     version: str
     git_sha: str | None
+    # Which channel resolved git_sha: "baked" (build-info file), "env"
+    # (ENDPOINTS_GIT_SHA), "git" (live checkout), or "none". Records how much to
+    # trust the SHA — a baked/committed SHA is ground truth, an env one an
+    # attestation, a dirty "git" one a warning. See utils/version.resolve_git_sha.
+    git_sha_source: str
     test_started_at: int
     n_samples_issued: int
     # Terminal responses; failed samples are a subset of this count.
@@ -365,6 +370,7 @@ def _series_dict(key: str) -> dict[str, Any]:
         return cls(
             version=str(version_info.get("version", "unknown")),
             git_sha=version_info.get("git_sha"),
+            git_sha_source=str(version_info.get("git_sha_source", "none")),
             test_started_at=0,  # TODO: surface session_started_ns via snapshot
             n_samples_issued=n_issued,
             n_samples_completed=n_completed,
@@ -423,8 +429,12 @@ def display(
                 f"final snapshot received) — some async metrics may be missing.{newline}"
             )
         fn(f"Version: {self.version}{newline}")
-        if self.git_sha:
-            fn(f"Git SHA: {self.git_sha}{newline}")
+        # Always emitted (even when unresolved) so a missing SHA is unambiguous —
+        # a blank line could mean capture failed or an old report format; the
+        # explicit "unknown (source: none)" says provenance was attempted.
+        fn(
+            f"Git SHA: {self.git_sha or 'unknown'} (source: {self.git_sha_source}){newline}"
+        )
         if self.run_config:
             fn(f"Run config:{newline}")
             for section, params in self.run_config.items():