feat(aiperf): profile existing GB300 vision-language models

CMakeLists.txt

@@ -552,6 +552,13 @@ if(TRTMC_BUILD_TESTS)
     target_compile_options(test_server_detection_worker PRIVATE -Wall -Wextra -Wpedantic)
     add_test(NAME server_detection_worker COMMAND test_server_detection_worker)
     set_tests_properties(server_detection_worker PROPERTIES LABELS cpu)
+    add_executable(test_server_vision_worker apps/server/tests/test_vision_worker.cpp)
+    target_include_directories(test_server_vision_worker PRIVATE
+      ${PROJECT_SOURCE_DIR}/core/runtime/include ${PROJECT_SOURCE_DIR}/apps)
+    target_link_libraries(test_server_vision_worker PRIVATE trtmc_server trtmc_media_io nlohmann_json::nlohmann_json)
+    target_compile_options(test_server_vision_worker PRIVATE -Wall -Wextra -Wpedantic)
+    add_test(NAME server_vision_worker COMMAND test_server_vision_worker)
+    set_tests_properties(server_vision_worker PROPERTIES LABELS cpu)
   endif()
 
   add_library(trtmc_test_cli_fixture SHARED apps/cli/tests/test_family_cli.cpp)
@@ -725,6 +732,7 @@ if(TRTMC_BUILD_TESTS)
     add_dependencies(test_server_sdk_worker trtmc-server trtmc_api_test_backend
       trtmc_api_test_family trtmc_api_test_text_family trtmc_api_test_stream_family
       trtmc_api_test_audio_family trtmc_api_test_image_boxes_family)
+    add_dependencies(test_server_sdk_worker trtmc_api_test_language_family)
     add_test(NAME server_sdk_worker
       COMMAND test_server_sdk_worker "${_trtmc_api_test_runtime_root}")
     add_test(NAME server_sdk_worker_process

apps/aiperf_qual/README.md

@@ -404,7 +404,7 @@ PYTHONPATH=apps/aiperf_qual:apps/benchmark:core/builder:. \
   python -m trtmc_aiperf_qual endpoint-models --repo . --group embeddings
 ```
 
-Groups are `embeddings`, `reranking`, `transcription`, and `detection`.
+Groups are `embeddings`, `reranking`, `transcription`, `detection`, and `vision`.
 The inventory selects a normal full-model profile when one is available, with
 precision and parallel variants listed separately. Checkpoints without one
 remain explicit gaps. Discovery reports `unverified`; it does not claim
@@ -437,17 +437,41 @@ PYTHONPATH=apps/aiperf_qual:apps/benchmark:apps/perf_serving:core/builder:. \
     --out /path/to/new-result-directory
 ```
 
+The default client tokenizer uses the profile's pinned checkpoint revision.
+Use `--tokenizer /path/to/prepared-tokenizer` when its files live separately;
+`--tokenizer-revision` pins an explicitly selected Hugging Face tokenizer.
+Hub overrides require a full commit SHA. Local tokenizer directories are
+fingerprinted before and after profiling, including their file contents.
+Lance and Phi4 provide `families/<family>/tests/aiperf/prepare_tokenizer.py`
+to export their existing tokenizers into a standalone client directory.
+Source identity is captured after acquiring the GPU lock, including content
+fingerprints for tracked changes and untracked files. Generated run output is
+excluded. Changes to source, tokenizer, bundle or inputs between the snapshots
+preserve raw evidence and prevent a success summary.
+
 Use the existing sentence/ranking examples or prepared audio/image JSONL from
 the guides below. The command verifies every corpus input against native output,
 then checks all 55 complete responses from five warmups and 50 measured requests,
-including SHA-256 hashes of media bytes, complete vectors/transcripts/boxes,
+including SHA-256 hashes of media bytes, complete vectors/transcripts/boxes/answers,
 native class IDs or passage indices, and every request/timing join. Results
 separate HTTP latency from public Task time. Build/load/startup, native/HTTP verification and
 warmup are excluded from measured throughput. This transport verification is
 separate from the owning family's numerical/quality criteria. It establishes
 no capacity or performance regression gate. The client is AIPerf 0.13.0 from
 PyPI; an AIPerf source checkout is unnecessary.
 
+DeepSeek-OCR-2 preserves aspect ratio with mean-color padding in its existing
+768-pixel, 145-visual-token single-view profile. Rebuild older square-warping
+bundles with the current family runtime. Its unchanged owning reference uses
+1024-pixel crop-mode inference; record that different visual budget when
+interpreting conversion parity. This endpoint run measures the native Task
+and does not claim a reference-relative speedup.
+
+Native InternVL3 retains one 448-pixel, 256-visual-token view. Its unchanged
+owning reference may tile one image into multiple views. Record that visual
+budget difference alongside conversion parity; serving one original image
+does not establish identical internal view counts.
+
 ## Native endpoint pilot coverage
 
 These original deliveries are historical pilot evidence. Use the stage issues
@@ -494,7 +518,8 @@ through AIPerf. This qualifies object detection; document/table layout remains p
 Use [Profile Embeddings with AIPerf](../../website/docs/user-guides/profile-embeddings-with-aiperf.md),
 [Profile Reranking with AIPerf](../../website/docs/user-guides/profile-reranking-with-aiperf.md),
 [Profile Transcription with AIPerf](../../website/docs/user-guides/profile-transcription-with-aiperf.md),
-or [Profile Object Detection with AIPerf](../../website/docs/user-guides/profile-detection-with-aiperf.md)
+[Profile Object Detection with AIPerf](../../website/docs/user-guides/profile-detection-with-aiperf.md),
+or [Profile Image Understanding with AIPerf](../../website/docs/user-guides/profile-vision-with-aiperf.md)
 for the short recipes. Native Task-call wall time stays separate from client
 HTTP latency; these small runs are integration examples, not performance gates.
 

apps/aiperf_qual/examples/prepare_vision_input.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Write one image/question pair for AIPerf without recompressing the image."""
+
+import argparse
+import base64
+import io
+import json
+from pathlib import Path
+
+from PIL import Image
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--image", type=Path, required=True)
+    parser.add_argument("--prompt", required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    raw = args.image.read_bytes()
+    with Image.open(io.BytesIO(raw)) as image:
+        if image.mode != "RGB" or image.format not in {"PNG", "JPEG"} or getattr(image, "n_frames", 1) != 1:
+            parser.error("image must be one RGB PNG or JPEG frame")
+        mime = "jpeg" if image.format == "JPEG" else "png"
+        image.verify()
+    if not args.prompt.strip():
+        parser.error("prompt must not be empty")
+    row = {"session_id": "image-question", "texts": [args.prompt],
+           "image": f"data:image/{mime};base64," + base64.b64encode(raw).decode("ascii")}
+    with args.output.open("x") as output:
+        output.write(json.dumps(row) + "\n")
+
+
+if __name__ == "__main__":
+    main()

apps/aiperf_qual/tests/test_endpoint_profile.py

@@ -3,6 +3,7 @@
 
 import base64
 import json
+from contextlib import nullcontext
 from pathlib import Path
 from types import SimpleNamespace
 
@@ -26,6 +27,25 @@ def test_invalid_or_nonendpoint_profile_has_a_clear_selection_error(tmp_path, mo
     assert not (tmp_path / "uncreated").exists()
 
 
+@pytest.mark.parametrize("revision", ["main", "v1.0", ""])
+def test_revision_only_override_is_validated_before_starting_a_profile(tmp_path, monkeypatch, revision):
+    import trtmc_benchmark.catalog as catalog
+    from trtmc_aiperf_qual import endpoint_profile
+
+    entry = SimpleNamespace(name="vision", hf_id="organization/checkpoint",
+                            model=SimpleNamespace(task="vision_language_generation", hf_revision="a" * 40))
+    monkeypatch.setattr(catalog, "ManifestCatalog", lambda _: SimpleNamespace(entries=lambda: [entry]))
+    monkeypatch.setattr(endpoint_profile, "inventory", lambda *_: [{"profile": "vision"}])
+    monkeypatch.setattr(endpoint_profile, "gpu_exclusive", lambda _: nullcontext())
+    monkeypatch.setattr(endpoint_profile, "source_identity",
+                        lambda *_: pytest.fail("invalid tokenizer revision reached profiling"))
+    environment = Environment({"repo": str(tmp_path), "runtime_root": str(tmp_path),
+                               "aiperf_python": "unused"})
+    with pytest.raises(ConfigError, match="full commit SHA"):
+        endpoint_profile.profile(environment, "vision", Path("unused"), Path("unused"),
+                                 tmp_path / "out", tokenizer_revision=revision)
+
+
 def test_singleton_embedding_wire_forms_keep_the_same_input_identity():
     assert input_key("embeddings", {"input": "hello"}) == input_key("embeddings", {"input": ["hello"]})
     with pytest.raises(ValueError):
@@ -87,3 +107,32 @@ def test_transcription_check_requires_the_complete_native_text():
     assert check_output("transcription", {"text": "Full transcript."}, {"text": "Full transcript."}) == 1
     with pytest.raises(ValueError):
         check_output("transcription", {"text": "Full"}, {"text": "Full transcript."})
+
+
+def test_vision_checks_complete_answers_token_counts_and_image_question_identity(tmp_path):
+    image = "data:image/png;base64," + base64.b64encode(b"original image").decode()
+    inputs = tmp_path / "inputs.jsonl"
+    inputs.write_text(json.dumps({"texts": ["Question?"], "image": image}) + "\n")
+    row, = samples("vision", inputs, "model", 10)
+    payload = row["payload"]
+    other = json.loads(json.dumps(payload))
+    other["messages"][0]["content"][0]["text"] = "Different question?"
+    assert input_key("vision", payload) != input_key("vision", other)
+    other = json.loads(json.dumps(payload))
+    other["messages"][0]["content"][1]["image_url"]["url"] = image + "AA=="
+    with pytest.raises(ValueError):
+        input_key("vision", other)
+    native = {"text": "Full answer.", "output_tokens": 3}
+    response = {"choices": [{"index": 0, "message": {"role": "assistant", "content": "Full answer."}}],
+                "usage": {"completion_tokens": 3}}
+    assert check_output("vision", response, native) == len(native["text"])
+    response["choices"][0]["message"]["content"] = "Full"
+    with pytest.raises(ValueError):
+        check_output("vision", response, native)
+    response["choices"][0]["message"]["content"] = ""
+    with pytest.raises(ValueError):
+        check_output("vision", response, {"text": "", "output_tokens": 3})
+    response["choices"][0]["message"]["content"] = "Full answer."
+    response["usage"]["completion_tokens"] = 2
+    with pytest.raises(ValueError):
+        check_output("vision", response, native)

apps/aiperf_qual/tests/test_endpoint_provenance.py

@@ -0,0 +1,74 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+
+import pytest
+
+from trtmc_aiperf_qual.config import ConfigError
+from trtmc_aiperf_qual.endpoint_profile import source_identity, tokenizer_identity
+
+
+@pytest.fixture
+def repository(tmp_path):
+    root = tmp_path / "repository"
+    root.mkdir()
+    for arguments in (["init", "-q"], ["config", "user.name", "Test"],
+                      ["config", "user.email", "test@example.com"]):
+        subprocess.run(["git", *arguments], cwd=root, check=True, capture_output=True)
+    (root / "source.py").write_text("value = 0\n")
+    subprocess.run(["git", "add", "."], cwd=root, check=True)
+    subprocess.run(["git", "commit", "--no-gpg-sign", "-qm", "fixture"], cwd=root, check=True)
+    return root
+
+
+def test_changes_to_an_already_dirty_tracked_file_change_identity(repository):
+    source = repository / "source.py"
+    source.write_text("value = 1\n")
+    before = source_identity(repository, repository / "output")
+    source.write_text("value = 2\n")
+    after = source_identity(repository, repository / "output")
+    assert before["source_head"] == after["source_head"]
+    assert before["source_status"] == after["source_status"]
+    assert before["source_contents_sha256"] != after["source_contents_sha256"]
+
+
+def test_untracked_source_content_changes_are_detected(repository):
+    source = repository / "new_source.py"
+    source.write_text("value = 1\n")
+    before = source_identity(repository, repository / "output")
+    source.write_text("value = 2\n")
+    after = source_identity(repository, repository / "output")
+    assert before["source_status"] == after["source_status"]
+    assert before["source_contents_sha256"] != after["source_contents_sha256"]
+
+
+def test_generated_output_does_not_change_source_identity(repository):
+    output = repository / "output [run]"
+    output.mkdir()
+    before = source_identity(repository, output)
+    (output / "nested").mkdir()
+    (output / "nested/records.json").write_text('{"requests": 55}\n')
+    assert source_identity(repository, output) == before
+
+
+def test_local_tokenizer_files_are_identified_by_contents(tmp_path):
+    directory = tmp_path / "tokenizer"
+    directory.mkdir()
+    config = directory / "tokenizer.json"
+    config.write_text('{"version": 1}\n')
+    before = tokenizer_identity(str(directory), None, override=True)
+    config.write_text('{"version": 2}\n')
+    after = tokenizer_identity(str(directory), None, override=True)
+    assert before["name"] == after["name"] and before["revision"] is after["revision"] is None
+    assert before["files_sha256"] != after["files_sha256"]
+
+
+@pytest.mark.parametrize("revision", [None, "main", "v1.0"])
+def test_remote_tokenizer_override_requires_an_immutable_revision(revision):
+    with pytest.raises(ConfigError, match="full commit SHA"):
+        tokenizer_identity("organization/tokenizer", revision, override=True)
+    pin = "a" * 40
+    assert tokenizer_identity("organization/tokenizer", pin, override=True) == {
+        "name": "organization/tokenizer", "revision": pin,
+    }

apps/aiperf_qual/trtmc_aiperf_qual/cli.py

@@ -430,22 +430,25 @@ def main(argv: Sequence[str] | None = None) -> int:
     text.add_argument("--out", type=Path, required=True)
     endpoints = commands.add_parser("endpoint-models", help="inventory every checkpoint for a native endpoint group")
     endpoints.add_argument("--repo", type=Path, required=True)
-    endpoints.add_argument("--group", choices=("embeddings", "reranking", "transcription", "detection"), required=True)
+    endpoints.add_argument("--group", choices=("embeddings", "reranking", "transcription", "detection", "vision"), required=True)
     endpoint_run = commands.add_parser("endpoint-profile", help="verify and profile one built native endpoint model")
     endpoint_run.add_argument("--environment", type=Path, required=True)
     endpoint_run.add_argument("--profile", required=True)
     endpoint_run.add_argument("--bundle", type=Path, required=True)
     endpoint_run.add_argument("--inputs", type=Path, required=True)
     endpoint_run.add_argument("--out", type=Path, required=True)
     endpoint_run.add_argument("--max-output-tokens", type=int, default=120)
+    endpoint_run.add_argument("--tokenizer", help="AIPerf client tokenizer name or prepared local directory")
+    endpoint_run.add_argument("--tokenizer-revision", help="revision of an explicitly selected client tokenizer")
     arguments = parser.parse_args(argv)
     try:
         if arguments.command == "endpoint-profile":
             from .endpoint_profile import profile as endpoint_profile
 
             result = endpoint_profile(load_environment(arguments.environment), arguments.profile,
                                       arguments.bundle, arguments.inputs, arguments.out,
-                                      token_limit=arguments.max_output_tokens)
+                                      token_limit=arguments.max_output_tokens, tokenizer=arguments.tokenizer,
+                                      tokenizer_revision=arguments.tokenizer_revision)
             print(json.dumps(result, indent=2))
             return 0
         if arguments.command == "endpoint-models":

apps/aiperf_qual/trtmc_aiperf_qual/endpoint_catalog.py

@@ -24,6 +24,7 @@
     "transcription": {"transcription", "speech_transcription", "transcription_streaming",
                       "streaming_speech_transcription"},
     "detection": {"object_detection", "image_to_boxes"},
+    "vision": {"vision_language_generation", "images_text_to_text"},
 }