From e7893c02b5a45fe7556e3f2352d3585b5942c55a Mon Sep 17 00:00:00 2001 From: Yizhuo Zhang Date: Fri, 16 Oct 2026 22:11:27 -0700 Subject: [PATCH 1/9] feat(aiperf): qualify native single-image language workloads Route image chat through the declared native Task, preserve full output and token counts, and verify request identity with source and tokenizer provenance. Keep preprocessing and checkpoint corrections with each owner. Cover the full native InternVL3 and 3.5 checkpoint set, including optional dynamic image tiles, alongside Qwen-VL, Phi4, Lance, LocateAnything and DeepSeek OCR. Preserve all existing correctness gates and record remaining Edge-backed coverage gaps. Inherit the independent Phi4 tokenizer and staged-plan memory fixes from the prerequisite. The integration itself changes five model families, within the existing public CI limit. Signed-off-by: Yizhuo Zhang --- CMakeLists.txt | 8 + apps/aiperf_qual/README.md | 31 ++- .../examples/prepare_vision_input.py | 37 +++ .../tests/test_endpoint_profile.py | 29 ++ .../tests/test_endpoint_provenance.py | 74 +++++ apps/aiperf_qual/trtmc_aiperf_qual/cli.py | 7 +- .../trtmc_aiperf_qual/endpoint_catalog.py | 1 + .../trtmc_aiperf_qual/endpoint_profile.py | 123 ++++++++- apps/server/native_worker.cpp | 82 ++++++ apps/server/python/tests/test_vision.py | 172 ++++++++++++ apps/server/python/trtmc_server/app.py | 61 ++++- apps/server/python/trtmc_server/cli.py | 2 +- apps/server/python/trtmc_server/protocol.py | 26 +- apps/server/python/trtmc_server/registry.py | 2 +- apps/server/python/trtmc_server/schemas.py | 11 +- apps/server/tests/test_sdk_worker.cpp | 45 +++ apps/server/tests/test_vision_worker.cpp | 102 +++++++ core/runtime/tensorrt/trt_module_impl.cpp | 3 +- .../tests/test_trt_module_dynamic_input.cpp | 29 ++ families/deepseek_ocr/model.py | 2 +- families/deepseek_ocr/runtime/CMakeLists.txt | 15 + .../runtime/image_preprocessor.cpp | 27 +- .../tests/cpp/test_image_padding.cpp | 55 ++++ .../tests/test_builder_contract.py | 4 +- .../deepseek_ocr/tests/test_vision_oracle.py | 12 + families/deepseek_ocr/tests/vision_oracle.py | 10 +- families/internvl/model.py | 22 +- .../tests/cpp/test_image_normalization.cpp | 40 +++ .../tests/manifests/internvl3-14b.json | 26 ++ .../tests/manifests/internvl3-1b.json | 22 ++ .../tests/manifests/internvl35-14b.json | 26 ++ .../tests/manifests/internvl35-1b.json | 22 ++ .../tests/manifests/internvl35-2b.json | 22 ++ .../tests/manifests/internvl35-4b.json | 22 ++ .../tests/manifests/internvl35-8b.json | 26 ++ families/internvl/tests/test_e2e.py | 47 ++-- families/internvl/tests/test_plan_lifetime.py | 79 ++++++ .../internvl/tests/test_prompt_framing.py | 41 +++ .../tests/test_qwen3_weight_mapping.py | 49 ++++ families/internvl/tests/test_tp_contract.py | 28 ++ families/internvl/tests/test_vision_oracle.py | 2 + .../tests/thresholds/internvl3-14b.json | 6 + .../tests/thresholds/internvl3-1b.json | 6 + .../tests/thresholds/internvl35-14b.json | 6 + .../tests/thresholds/internvl35-1b.json | 6 + .../tests/thresholds/internvl35-2b.json | 6 + .../tests/thresholds/internvl35-4b.json | 6 + .../tests/thresholds/internvl35-8b.json | 6 + .../lance/tests/aiperf/prepare_tokenizer.py | 47 ++++ families/locateanything/model.py | 20 +- families/locateanything/tests/hf_reference.py | 13 +- .../locateanything/tests/test_hf_reference.py | 17 +- .../tests/test_tokenizer_packaging.py | 39 +++ families/qwen_vl/model.py | 41 ++- families/qwen_vl/qwen_vl_vision_builder.py | 259 ++++++++++++------ .../qwen_vl/runtime/image_preprocessor.cpp | 48 +++- families/qwen_vl/runtime/image_preprocessor.h | 3 + families/qwen_vl/runtime/pipeline.cpp | 6 + .../tests/cpp/test_image_normalization.cpp | 67 +++++ .../tests/test_bf16_precision_contract.py | 38 +++ .../tests/test_dynamic_window_profile.py | 51 ++++ .../qwen_vl/tests/test_image_normalization.py | 29 +- families/qwen_vl/tests/vision_oracle.py | 85 +++++- .../user-guides/profile-vision-with-aiperf.md | 110 ++++++++ .../docs/user-guides/serve-text-generation.md | 12 +- website/sidebars.js | 1 + 66 files changed, 2179 insertions(+), 193 deletions(-) create mode 100644 apps/aiperf_qual/examples/prepare_vision_input.py create mode 100644 apps/aiperf_qual/tests/test_endpoint_provenance.py create mode 100644 apps/server/python/tests/test_vision.py create mode 100644 apps/server/tests/test_vision_worker.cpp create mode 100644 families/deepseek_ocr/tests/cpp/test_image_padding.cpp create mode 100644 families/internvl/tests/manifests/internvl3-14b.json create mode 100644 families/internvl/tests/manifests/internvl3-1b.json create mode 100644 families/internvl/tests/manifests/internvl35-14b.json create mode 100644 families/internvl/tests/manifests/internvl35-1b.json create mode 100644 families/internvl/tests/manifests/internvl35-2b.json create mode 100644 families/internvl/tests/manifests/internvl35-4b.json create mode 100644 families/internvl/tests/manifests/internvl35-8b.json create mode 100644 families/internvl/tests/test_plan_lifetime.py create mode 100644 families/internvl/tests/test_prompt_framing.py create mode 100644 families/internvl/tests/test_qwen3_weight_mapping.py create mode 100644 families/internvl/tests/thresholds/internvl3-14b.json create mode 100644 families/internvl/tests/thresholds/internvl3-1b.json create mode 100644 families/internvl/tests/thresholds/internvl35-14b.json create mode 100644 families/internvl/tests/thresholds/internvl35-1b.json create mode 100644 families/internvl/tests/thresholds/internvl35-2b.json create mode 100644 families/internvl/tests/thresholds/internvl35-4b.json create mode 100644 families/internvl/tests/thresholds/internvl35-8b.json create mode 100644 families/lance/tests/aiperf/prepare_tokenizer.py create mode 100644 families/locateanything/tests/test_tokenizer_packaging.py create mode 100644 families/qwen_vl/tests/test_dynamic_window_profile.py create mode 100644 website/docs/user-guides/profile-vision-with-aiperf.md diff --git a/CMakeLists.txt b/CMakeLists.txt index e4e60c61d8..185761ff78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -552,6 +552,13 @@ if(TRTMC_BUILD_TESTS) target_compile_options(test_server_detection_worker PRIVATE -Wall -Wextra -Wpedantic) add_test(NAME server_detection_worker COMMAND test_server_detection_worker) set_tests_properties(server_detection_worker PROPERTIES LABELS cpu) + add_executable(test_server_vision_worker apps/server/tests/test_vision_worker.cpp) + target_include_directories(test_server_vision_worker PRIVATE + ${PROJECT_SOURCE_DIR}/core/runtime/include ${PROJECT_SOURCE_DIR}/apps) + target_link_libraries(test_server_vision_worker PRIVATE trtmc_server trtmc_media_io nlohmann_json::nlohmann_json) + target_compile_options(test_server_vision_worker PRIVATE -Wall -Wextra -Wpedantic) + add_test(NAME server_vision_worker COMMAND test_server_vision_worker) + set_tests_properties(server_vision_worker PROPERTIES LABELS cpu) endif() add_library(trtmc_test_cli_fixture SHARED apps/cli/tests/test_family_cli.cpp) @@ -725,6 +732,7 @@ if(TRTMC_BUILD_TESTS) add_dependencies(test_server_sdk_worker trtmc-server trtmc_api_test_backend trtmc_api_test_family trtmc_api_test_text_family trtmc_api_test_stream_family trtmc_api_test_audio_family trtmc_api_test_image_boxes_family) + add_dependencies(test_server_sdk_worker trtmc_api_test_language_family) add_test(NAME server_sdk_worker COMMAND test_server_sdk_worker "${_trtmc_api_test_runtime_root}") add_test(NAME server_sdk_worker_process diff --git a/apps/aiperf_qual/README.md b/apps/aiperf_qual/README.md index d88e6c934a..6705d0b9f9 100644 --- a/apps/aiperf_qual/README.md +++ b/apps/aiperf_qual/README.md @@ -404,7 +404,7 @@ PYTHONPATH=apps/aiperf_qual:apps/benchmark:core/builder:. \ python -m trtmc_aiperf_qual endpoint-models --repo . --group embeddings ``` -Groups are `embeddings`, `reranking`, `transcription`, and `detection`. +Groups are `embeddings`, `reranking`, `transcription`, `detection`, and `vision`. The inventory selects a normal full-model profile when one is available, with precision and parallel variants listed separately. Checkpoints without one remain explicit gaps. Discovery reports `unverified`; it does not claim @@ -437,10 +437,22 @@ PYTHONPATH=apps/aiperf_qual:apps/benchmark:apps/perf_serving:core/builder:. \ --out /path/to/new-result-directory ``` +The default client tokenizer uses the profile's pinned checkpoint revision. +Use `--tokenizer /path/to/prepared-tokenizer` when its files live separately; +`--tokenizer-revision` pins an explicitly selected Hugging Face tokenizer. +Hub overrides require a full commit SHA. Local tokenizer directories are +fingerprinted before and after profiling, including their file contents. +Lance and Phi4 provide `families//tests/aiperf/prepare_tokenizer.py` +to export their existing tokenizers into a standalone client directory. +Source identity is captured after acquiring the GPU lock, including content +fingerprints for tracked changes and untracked files. Generated run output is +excluded. Changes to source, tokenizer, bundle or inputs between the snapshots +preserve raw evidence and prevent a success summary. + Use the existing sentence/ranking examples or prepared audio/image JSONL from the guides below. The command verifies every corpus input against native output, then checks all 55 complete responses from five warmups and 50 measured requests, -including SHA-256 hashes of media bytes, complete vectors/transcripts/boxes, +including SHA-256 hashes of media bytes, complete vectors/transcripts/boxes/answers, native class IDs or passage indices, and every request/timing join. Results separate HTTP latency from public Task time. Build/load/startup, native/HTTP verification and warmup are excluded from measured throughput. This transport verification is @@ -448,6 +460,18 @@ separate from the owning family's numerical/quality criteria. It establishes no capacity or performance regression gate. The client is AIPerf 0.13.0 from PyPI; an AIPerf source checkout is unnecessary. +DeepSeek-OCR-2 preserves aspect ratio with mean-color padding in its existing +768-pixel, 145-visual-token single-view profile. Rebuild older square-warping +bundles with the current family runtime. Its unchanged owning reference uses +1024-pixel crop-mode inference; record that different visual budget when +interpreting conversion parity. This endpoint run measures the native Task +and does not claim a reference-relative speedup. + +Native InternVL3 retains one 448-pixel, 256-visual-token view. Its unchanged +owning reference may tile one image into multiple views. Record that visual +budget difference alongside conversion parity; serving one original image +does not establish identical internal view counts. + ## Native endpoint pilot coverage These original deliveries are historical pilot evidence. Use the stage issues @@ -494,7 +518,8 @@ through AIPerf. This qualifies object detection; document/table layout remains p Use [Profile Embeddings with AIPerf](../../website/docs/user-guides/profile-embeddings-with-aiperf.md), [Profile Reranking with AIPerf](../../website/docs/user-guides/profile-reranking-with-aiperf.md), [Profile Transcription with AIPerf](../../website/docs/user-guides/profile-transcription-with-aiperf.md), -or [Profile Object Detection with AIPerf](../../website/docs/user-guides/profile-detection-with-aiperf.md) +[Profile Object Detection with AIPerf](../../website/docs/user-guides/profile-detection-with-aiperf.md), +or [Profile Image Understanding with AIPerf](../../website/docs/user-guides/profile-vision-with-aiperf.md) for the short recipes. Native Task-call wall time stays separate from client HTTP latency; these small runs are integration examples, not performance gates. diff --git a/apps/aiperf_qual/examples/prepare_vision_input.py b/apps/aiperf_qual/examples/prepare_vision_input.py new file mode 100644 index 0000000000..ff40010d3c --- /dev/null +++ b/apps/aiperf_qual/examples/prepare_vision_input.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Write one image/question pair for AIPerf without recompressing the image.""" + +import argparse +import base64 +import io +import json +from pathlib import Path + +from PIL import Image + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image", type=Path, required=True) + parser.add_argument("--prompt", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + raw = args.image.read_bytes() + with Image.open(io.BytesIO(raw)) as image: + if image.mode != "RGB" or image.format not in {"PNG", "JPEG"} or getattr(image, "n_frames", 1) != 1: + parser.error("image must be one RGB PNG or JPEG frame") + mime = "jpeg" if image.format == "JPEG" else "png" + image.verify() + if not args.prompt.strip(): + parser.error("prompt must not be empty") + row = {"session_id": "image-question", "texts": [args.prompt], + "image": f"data:image/{mime};base64," + base64.b64encode(raw).decode("ascii")} + with args.output.open("x") as output: + output.write(json.dumps(row) + "\n") + + +if __name__ == "__main__": + main() diff --git a/apps/aiperf_qual/tests/test_endpoint_profile.py b/apps/aiperf_qual/tests/test_endpoint_profile.py index dc3ca8b3d1..1f1902865a 100644 --- a/apps/aiperf_qual/tests/test_endpoint_profile.py +++ b/apps/aiperf_qual/tests/test_endpoint_profile.py @@ -87,3 +87,32 @@ def test_transcription_check_requires_the_complete_native_text(): assert check_output("transcription", {"text": "Full transcript."}, {"text": "Full transcript."}) == 1 with pytest.raises(ValueError): check_output("transcription", {"text": "Full"}, {"text": "Full transcript."}) + + +def test_vision_checks_complete_answers_token_counts_and_image_question_identity(tmp_path): + image = "data:image/png;base64," + base64.b64encode(b"original image").decode() + inputs = tmp_path / "inputs.jsonl" + inputs.write_text(json.dumps({"texts": ["Question?"], "image": image}) + "\n") + row, = samples("vision", inputs, "model", 10) + payload = row["payload"] + other = json.loads(json.dumps(payload)) + other["messages"][0]["content"][0]["text"] = "Different question?" + assert input_key("vision", payload) != input_key("vision", other) + other = json.loads(json.dumps(payload)) + other["messages"][0]["content"][1]["image_url"]["url"] = image + "AA==" + with pytest.raises(ValueError): + input_key("vision", other) + native = {"text": "Full answer.", "output_tokens": 3} + response = {"choices": [{"index": 0, "message": {"role": "assistant", "content": "Full answer."}}], + "usage": {"completion_tokens": 3}} + assert check_output("vision", response, native) == len(native["text"]) + response["choices"][0]["message"]["content"] = "Full" + with pytest.raises(ValueError): + check_output("vision", response, native) + response["choices"][0]["message"]["content"] = "" + with pytest.raises(ValueError): + check_output("vision", response, {"text": "", "output_tokens": 3}) + response["choices"][0]["message"]["content"] = "Full answer." + response["usage"]["completion_tokens"] = 2 + with pytest.raises(ValueError): + check_output("vision", response, native) diff --git a/apps/aiperf_qual/tests/test_endpoint_provenance.py b/apps/aiperf_qual/tests/test_endpoint_provenance.py new file mode 100644 index 0000000000..2755453ba2 --- /dev/null +++ b/apps/aiperf_qual/tests/test_endpoint_provenance.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import subprocess + +import pytest + +from trtmc_aiperf_qual.config import ConfigError +from trtmc_aiperf_qual.endpoint_profile import source_identity, tokenizer_identity + + +@pytest.fixture +def repository(tmp_path): + root = tmp_path / "repository" + root.mkdir() + for arguments in (["init", "-q"], ["config", "user.name", "Test"], + ["config", "user.email", "test@example.com"]): + subprocess.run(["git", *arguments], cwd=root, check=True, capture_output=True) + (root / "source.py").write_text("value = 0\n") + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "--no-gpg-sign", "-qm", "fixture"], cwd=root, check=True) + return root + + +def test_changes_to_an_already_dirty_tracked_file_change_identity(repository): + source = repository / "source.py" + source.write_text("value = 1\n") + before = source_identity(repository, repository / "output") + source.write_text("value = 2\n") + after = source_identity(repository, repository / "output") + assert before["source_head"] == after["source_head"] + assert before["source_status"] == after["source_status"] + assert before["source_contents_sha256"] != after["source_contents_sha256"] + + +def test_untracked_source_content_changes_are_detected(repository): + source = repository / "new_source.py" + source.write_text("value = 1\n") + before = source_identity(repository, repository / "output") + source.write_text("value = 2\n") + after = source_identity(repository, repository / "output") + assert before["source_status"] == after["source_status"] + assert before["source_contents_sha256"] != after["source_contents_sha256"] + + +def test_generated_output_does_not_change_source_identity(repository): + output = repository / "output [run]" + output.mkdir() + before = source_identity(repository, output) + (output / "nested").mkdir() + (output / "nested/records.json").write_text('{"requests": 55}\n') + assert source_identity(repository, output) == before + + +def test_local_tokenizer_files_are_identified_by_contents(tmp_path): + directory = tmp_path / "tokenizer" + directory.mkdir() + config = directory / "tokenizer.json" + config.write_text('{"version": 1}\n') + before = tokenizer_identity(str(directory), None, override=True) + config.write_text('{"version": 2}\n') + after = tokenizer_identity(str(directory), None, override=True) + assert before["name"] == after["name"] and before["revision"] is after["revision"] is None + assert before["files_sha256"] != after["files_sha256"] + + +@pytest.mark.parametrize("revision", [None, "main", "v1.0"]) +def test_remote_tokenizer_override_requires_an_immutable_revision(revision): + with pytest.raises(ConfigError, match="full commit SHA"): + tokenizer_identity("organization/tokenizer", revision, override=True) + pin = "a" * 40 + assert tokenizer_identity("organization/tokenizer", pin, override=True) == { + "name": "organization/tokenizer", "revision": pin, + } diff --git a/apps/aiperf_qual/trtmc_aiperf_qual/cli.py b/apps/aiperf_qual/trtmc_aiperf_qual/cli.py index 6478433e43..728aeffb9b 100644 --- a/apps/aiperf_qual/trtmc_aiperf_qual/cli.py +++ b/apps/aiperf_qual/trtmc_aiperf_qual/cli.py @@ -430,7 +430,7 @@ def main(argv: Sequence[str] | None = None) -> int: text.add_argument("--out", type=Path, required=True) endpoints = commands.add_parser("endpoint-models", help="inventory every checkpoint for a native endpoint group") endpoints.add_argument("--repo", type=Path, required=True) - endpoints.add_argument("--group", choices=("embeddings", "reranking", "transcription", "detection"), required=True) + endpoints.add_argument("--group", choices=("embeddings", "reranking", "transcription", "detection", "vision"), required=True) endpoint_run = commands.add_parser("endpoint-profile", help="verify and profile one built native endpoint model") endpoint_run.add_argument("--environment", type=Path, required=True) endpoint_run.add_argument("--profile", required=True) @@ -438,6 +438,8 @@ def main(argv: Sequence[str] | None = None) -> int: endpoint_run.add_argument("--inputs", type=Path, required=True) endpoint_run.add_argument("--out", type=Path, required=True) endpoint_run.add_argument("--max-output-tokens", type=int, default=120) + endpoint_run.add_argument("--tokenizer", help="AIPerf client tokenizer name or prepared local directory") + endpoint_run.add_argument("--tokenizer-revision", help="revision of an explicitly selected client tokenizer") arguments = parser.parse_args(argv) try: if arguments.command == "endpoint-profile": @@ -445,7 +447,8 @@ def main(argv: Sequence[str] | None = None) -> int: result = endpoint_profile(load_environment(arguments.environment), arguments.profile, arguments.bundle, arguments.inputs, arguments.out, - token_limit=arguments.max_output_tokens) + token_limit=arguments.max_output_tokens, tokenizer=arguments.tokenizer, + tokenizer_revision=arguments.tokenizer_revision) print(json.dumps(result, indent=2)) return 0 if arguments.command == "endpoint-models": diff --git a/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_catalog.py b/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_catalog.py index b633d522b2..507aaa181c 100644 --- a/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_catalog.py +++ b/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_catalog.py @@ -24,6 +24,7 @@ "transcription": {"transcription", "speech_transcription", "transcription_streaming", "streaming_speech_transcription"}, "detection": {"object_detection", "image_to_boxes"}, + "vision": {"vision_language_generation", "images_text_to_text"}, } diff --git a/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_profile.py b/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_profile.py index 94b79b1f2e..833e4c2695 100644 --- a/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_profile.py +++ b/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_profile.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Profile an existing native bundle through the standard non-generation endpoints. +"""Profile an existing native bundle through standard single-input endpoints. Reuse the persistent server, benchmark worker, published AIPerf runner, and request/metric joins. Family correctness remains a separate prerequisite; @@ -14,6 +14,9 @@ import hashlib import json import math +import os +import re +import subprocess import urllib.request import uuid from pathlib import Path @@ -31,9 +34,61 @@ "reranking": ("nim_rankings", "/v1/ranking"), "transcription": ("audio_transcription", "/v1/audio/transcriptions"), "detection": ("image_retrieval", "/v1/infer"), + "vision": ("chat", "/v1/chat/completions"), } +def source_identity(repository: Path, output: Path) -> dict[str, str]: + """Fingerprint Git contents, including an already-dirty working tree.""" + paths = ["--", "."] + try: + relative = output.resolve().relative_to(repository.resolve()) + except ValueError: + pass + else: + paths.append(f":(top,literal,exclude){relative.as_posix()}") + delta = subprocess.check_output( + ["git", "diff", "HEAD", "--binary", "--no-ext-diff", "--no-textconv", *paths], cwd=repository, + ) + untracked = subprocess.check_output( + ["git", "ls-files", "--others", "--exclude-standard", "-z", *paths], cwd=repository, + ) + files = {} + for raw in sorted(name for name in untracked.split(b"\0") if name): + name = os.fsdecode(raw) + path = repository / name + files[name] = {"link": os.readlink(path) if path.is_symlink() else None, + "sha256": digest(path) if path.is_file() else None} + contents = json.dumps({"tracked_delta": hashlib.sha256(delta).hexdigest(), "untracked": files}, + sort_keys=True).encode() + return {"source_head": git(repository, "rev-parse", "HEAD"), + "source_status": git(repository, "status", "--porcelain", "--untracked-files=all", *paths), + "source_contents_sha256": hashlib.sha256(contents).hexdigest()} + + +def tokenizer_identity(name: str, revision: str | None, *, override: bool) -> dict[str, Any]: + """Identify local tokenizer contents or an immutable Hub override.""" + identity = {"name": name, "revision": revision} + directory = Path(name).expanduser() + if directory.is_dir(): + files = {} + for path in sorted(directory.rglob("*")): + if path.is_symlink() and path.is_dir(): + raise ConfigError("local tokenizer directories cannot contain directory symlinks") + if path.is_dir(): + continue + if not path.is_file(): + raise ConfigError("local tokenizer contents must be readable regular files") + files[path.relative_to(directory).as_posix()] = { + "link": os.readlink(path) if path.is_symlink() else None, "sha256": digest(path), + } + identity.update(local_path=str(directory.resolve()), + files_sha256=hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest()) + elif override and (revision is None or re.fullmatch(r"[0-9a-fA-F]{40}", revision) is None): + raise ConfigError("a Hub tokenizer override requires --tokenizer-revision with a full commit SHA") + return identity + + def input_key(group: str, payload: dict[str, Any]) -> str: """Identify the effective wire input, including original media bytes.""" if group == "transcription": @@ -45,6 +100,13 @@ def input_key(group: str, payload: dict[str, Any]) -> str: if isinstance(text, list): text, = text value = json.dumps(text, ensure_ascii=False).encode() + elif group == "vision": + message, = payload["messages"] + parts = message["content"] + image, = [part["image_url"]["url"] for part in parts if part["type"] == "image_url"] + image_digest = hashlib.sha256(base64.b64decode(image.split(",", 1)[1], validate=True)).hexdigest() + text = "".join(part["text"] for part in parts if part["type"] == "text") + value = json.dumps([text, image_digest], ensure_ascii=False).encode() else: value = json.dumps([payload["query"], payload["passages"]], sort_keys=True, ensure_ascii=False).encode() return hashlib.sha256(value).hexdigest() @@ -71,6 +133,12 @@ def samples(group: str, inputs: Path, model: str, token_limit: int) -> list[dict payload = {"model": model, "max_output_tokens": token_limit, "file": {"b64_data": base64.b64encode(path.read_bytes()).decode()}} native = {"audio_path": str(path), "max_new_tokens": token_limit, "streaming": False} + elif group == "vision": + prompt, = row["texts"] + payload = {"model": model, "messages": [{"role": "user", "content": [ + {"type": "text", "text": prompt}, {"type": "image_url", "image_url": {"url": row["image"]}}]}], + "max_tokens": token_limit, "temperature": 1.0, "top_k": 1} + native = {"prompt": prompt, "max_new_tokens": token_limit, "temperature": 1.0, "top_k": 1} else: payload = {"input": [{"type": "image_url", "url": row["image"]}], "model": model} native = {} # The caller writes exactly these embedded bytes for the native image decoder. @@ -102,6 +170,14 @@ def post(url: str, payload: dict[str, Any], request_id: str, group: str) -> Any: def check_output(group: str, actual: Any, native: dict[str, Any], dimensions=None) -> int: """Preserve the complete public Task output, not a summary or a success code.""" + if group == "vision": + choice, = actual["choices"] + text = choice["message"]["content"] + if (not isinstance(text, str) or not text.strip() or text != native["text"] or + choice["message"]["role"] != "assistant" or choice["index"] != 0 or + actual["usage"]["completion_tokens"] != native["output_tokens"]): + raise ValueError("HTTP answer or token count differs from the complete native result") + return len(text) if group == "transcription": if actual["text"] != native["text"]: raise ValueError("HTTP transcript differs from the complete native transcript") @@ -150,7 +226,8 @@ def check_output(group: str, actual: Any, native: dict[str, Any], dimensions=Non def profile(environment: Environment, name: str, bundle: Path, inputs: Path, out: Path, - *, token_limit: int = 120) -> dict[str, Any]: + *, token_limit: int = 120, tokenizer: str | None = None, + tokenizer_revision: str | None = None) -> dict[str, Any]: """One serial model run: native verification, then 5 warmups and 50 measured requests.""" from trtmc_benchmark.catalog import ManifestCatalog from trtmc_perf_serving.backends.trtmc import TrtmcWorkerBackend @@ -165,16 +242,19 @@ def profile(environment: Environment, name: str, bundle: Path, inputs: Path, out raise ConfigError("select the group's normal full-model checkpoint profile") out.mkdir(parents=True, exist_ok=False) client = str(environment["aiperf_python"]) - client_info = check_aiperf(environment, {}) runtime = environment.path("runtime_root") - provenance = {"source_head": git(repo, "rev-parse", "HEAD"), - "source_status": git(repo, "status", "--porcelain"), "profile": name, - "manifest": json.loads(entry.model.manifest_path.read_text()), - "bundle_sha256": digest(bundle), "inputs_sha256": digest(inputs), "aiperf": client_info} - (out / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n") - corpus = samples(group, inputs, entry.hf_id, token_limit) expected = {} with gpu_exclusive(environment): + # Capture the source that actually starts inference, after any lock wait. + revision = tokenizer_revision or (entry.model.hf_revision if tokenizer is None else None) + selected_tokenizer = tokenizer_identity(tokenizer or entry.hf_id, revision, override=tokenizer is not None) + provenance = {**source_identity(repo, out), "profile": name, + "manifest": json.loads(entry.model.manifest_path.read_text()), + "bundle_sha256": digest(bundle), "inputs_sha256": digest(inputs), + "aiperf": check_aiperf(environment, {}), + "tokenizer": selected_tokenizer} + (out / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n") + corpus = samples(group, inputs, entry.hf_id, token_limit) native = TrtmcWorkerBackend(worker=runtime / "trtmc_benchmark_worker", session={ "schema_version": 2, "operation": entry.operation, "bundle": str(bundle), "runtime_root": str(runtime), "request": {}, "expected_family": entry.family, @@ -182,15 +262,17 @@ def profile(environment: Environment, name: str, bundle: Path, inputs: Path, out try: for row in corpus: dimensions = None - if group == "detection": + if group in {"detection", "vision"}: from PIL import Image - raw = base64.b64decode(row["payload"]["input"][0]["url"].split(",", 1)[1], validate=True) + raw = base64.b64decode((row["payload"]["input"][0]["url"] if group == "detection" + else next(part["image_url"]["url"] for part in row["payload"]["messages"][0]["content"] + if part["type"] == "image_url")).split(",", 1)[1], validate=True) path = out / (row["key"] + ".image") path.write_bytes(raw) with Image.open(path) as image: dimensions = image.size - row["request"] = {"image_path": str(path)} + row["request"]["image_path"] = str(path) observed = native.invoke(row["request"], out / row["key"]).observation expected[row["key"]] = {"observation": observed, "dimensions": dimensions} finally: @@ -209,6 +291,12 @@ def profile(environment: Environment, name: str, bundle: Path, inputs: Path, out "--warmup-request-count", "5", "--warmup-concurrency", "1", "--request-count", "50"] if group == "transcription": arguments += ["--extra-inputs", f"max_output_tokens:{token_limit}"] + if group == "vision": + arguments += ["--extra-inputs", f"max_tokens:{token_limit}", "temperature:1.0", "top_k:1"] + if tokenizer: + arguments += ["--tokenizer", selected_tokenizer.get("local_path", tokenizer)] + if revision: + arguments += ["--tokenizer-revision", revision] run = run_aiperf(environment, out / "client", arguments, model_name=entry.hf_id, command_prefix=[client, "-m", "aiperf"], timeout_s=1800) if run.exit_code: @@ -246,7 +334,16 @@ def profile(environment: Environment, name: str, bundle: Path, inputs: Path, out summary.update(measured_audio_seconds=seconds, audio_seconds_per_second=seconds / elapsed) if group == "reranking": summary["passages_per_second"] = sum(r["server"]["input_items"] for r in measured) / elapsed - if group == "detection": + if group in {"detection", "vision"}: summary["images_per_second"] = summary["requests_per_second"] + final_source = source_identity(repo, out) + provenance["final_source"] = final_source + final_tokenizer = tokenizer_identity(tokenizer or entry.hf_id, revision, override=tokenizer is not None) + provenance["final_tokenizer"] = final_tokenizer + (out / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n") + if (any(final_source[key] != provenance[key] for key in final_source) or + final_tokenizer != selected_tokenizer or + digest(bundle) != provenance["bundle_sha256"] or digest(inputs) != provenance["inputs_sha256"]): + raise ConfigError("source, tokenizer, bundle, or inputs changed during profiling; raw evidence is preserved") (out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") return summary diff --git a/apps/server/native_worker.cpp b/apps/server/native_worker.cpp index 4c2babcaf8..fb13bc5d53 100644 --- a/apps/server/native_worker.cpp +++ b/apps/server/native_worker.cpp @@ -11,6 +11,7 @@ #include "trtmc/audio.hpp" #include "trtmc/control.hpp" #include "trtmc/features.hpp" +#include "trtmc/language.hpp" #include "trtmc/perception.hpp" #include "trtmc/runtime/family_loader.h" #include "trtmc/stream.hpp" @@ -447,9 +448,62 @@ int run_detection_protocol(Invoke&& invoke, Extract&& extract, std::istream& inp input, output); } +template +int run_vision_protocol(std::int64_t default_tokens, Generate&& generate, std::istream& input, + std::ostream& output) { + return run_protocol({{"event", "ready"}, + {"protocol_version", 2}, + {"capabilities", Json::array({"vision_language_generation"})}, + {"default_max_new_tokens", default_tokens}}, + {{"generate_vision", + [&](const Json& request) { + const auto prompt = string_field(request, "prompt"); + cli::io::LoadedImage image; + const auto path = string_field(request, "image_path"); + try { + image = cli::io::read_image(path); + } catch (const std::exception&) { + throw ProtocolError("image must be a readable image file"); + } + auto result = generate(prompt, image, request); + result["image_height"] = image.height; + result["image_width"] = image.width; + return result; + }}}, + input, output); +} + int run_existing_worker(ITask& task, std::istream& input, std::ostream& output) { // Legacy bundle modes select the declared public Task before invocation. // Several families implement more C++ interfaces than a given bundle enables. + if (std::string_view(task.task()) == IVisionLanguageGeneration::kTask) { + auto* vision = dynamic_cast(&task); + if (!vision) + throw std::invalid_argument("bundle does not implement vision-language generation"); + const auto* text = dynamic_cast(&task); + const auto defaults = text ? text->default_max_new_tokens() : 128; + return run_vision_protocol( + defaults, + [&](const std::string& prompt, const cli::io::LoadedImage& image, const Json& request) { + const auto config = parse_config(request, defaults); + if (!config.system_prompt.empty()) + throw ProtocolError("system prompts are unsupported for image requests"); + const auto started = std::chrono::steady_clock::now(); + const auto result = vision->generate(prompt, image.pixels.data(), image.height, + image.width, config); + const auto wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started) + .count(); + return Json{{"text", result.text}, + {"completion_tokens", result.token_ids.size()}, + {"setup_ms", result.setup_ms}, + {"prefill_ms", result.prefill_ms}, + {"decode_ms", result.decode_ms}, + {"model_call_ms", wall_ms}, + {"timing_scope", "public_task_call_wall"}}; + }, + input, output); + } if (std::string_view(task.task()) == IEncoding::kTask) { auto* encoder = dynamic_cast(&task); if (!encoder) @@ -528,6 +582,34 @@ int run_existing_worker(ITask& task, std::istream& input, std::ostream& output) } int run_model_worker(const Model& model, std::istream& input, std::ostream& output) { + const auto primary = model.info().bundle_task; + if (primary == ImagesTextToText::kTask || primary == IVisionLanguageGeneration::kTask) { + const auto task = model.task(); + const auto fields = task.config_fields(); + return run_vision_protocol( + default_token_limit(fields), + [&](const std::string& prompt, const cli::io::LoadedImage& image, const Json& request) { + const auto config = parse_sdk_config(request, fields); + const auto input = ImagesTextToTextRequest::from_parts( + {ImageInput{{image.pixels.data(), image.pixels.size()}, + static_cast(image.height), + static_cast(image.width)}, + TextPart{prompt}}); + const auto started = std::chrono::steady_clock::now(); + const auto result = task.run(input, config); + const auto wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started) + .count(); + return Json{{"text", std::string(result.text())}, + {"completion_tokens", result.token_ids().size()}, + {"setup_ms", result.setup_ms()}, + {"prefill_ms", result.prefill_ms()}, + {"decode_ms", result.decode_ms()}, + {"model_call_ms", wall_ms}, + {"timing_scope", "public_task_call_wall"}}; + }, + input, output); + } if (model.supports()) { const auto task = model.task(); return run_embedding_protocol([&](const std::string& text) { return task.run({text}); }, diff --git a/apps/server/python/tests/test_vision.py b/apps/server/python/tests/test_vision.py new file mode 100644 index 0000000000..4c41228a97 --- /dev/null +++ b/apps/server/python/tests/test_vision.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import base64 +from concurrent.futures import Future +import io +import json +from pathlib import Path + +from fastapi.testclient import TestClient +from PIL import Image +import pytest + +from trtmc_server.app import ServerConfig, create_app +from trtmc_server.errors import WorkerCrashedError, WorkerSaturatedError +from trtmc_server.registry import ModelRegistry + + +class Registry: + def __init__(self): + data = io.BytesIO() + Image.new("RGB", (8, 4), color="red").save(data, format="PNG") + self.image = data.getvalue() + self.paths = [] + self.calls = 0 + self.closed = 0 + self.retired = False + self.failure = None + self.result = {"text": "Red. Complete answer.", "completion_tokens": 3, + "image_height": 4, "image_width": 8, + "model_call_ms": 12.5, "timing_scope": "public_task_call_wall"} + + def start(self): + pass + + def close(self): + pass + + def max_tokens(self, model, cap): + return min(10, cap) + + def supports(self, model, capability): + return model == "vision" and capability == "vision_language_generation" + + def acquire(self, model): + if self.failure == "busy": + raise WorkerSaturatedError("busy") + return Session(self) + + +class Session: + replica_name = "vision-1" + + def __init__(self, registry): + self.registry = registry + + def submit(self, operation, payload): + path = Path(payload["image_path"]) + self.registry.paths.append(path) + assert path.read_bytes() == self.registry.image + assert operation == "generate_vision" and payload["prompt"] == "Color?" + assert payload["config"] == {"max_new_tokens": 10, "temperature": 1.0, "top_k": 1} + self.registry.calls += 1 + if self.registry.failure == "crashed": + raise WorkerCrashedError("private path") + result = Future() + result.set_result(self.registry.result) + return result + + def close(self): + self.registry.closed += 1 + + def retire(self, error): + self.registry.retired = True + result = Future() + result.set_result(None) + return result + + +def payload(registry): + return {"model": "vision", "max_tokens": 10, "temperature": 1.0, "top_k": 1, + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "Color?"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + + base64.b64encode(registry.image).decode()}}]}]} + + +def client_for(registry, records=None, **limits): + return TestClient(create_app(registry, ServerConfig(max_body_bytes=16000, max_prompt_bytes=128, + max_generation_tokens=128, records=records, **limits))) + + +def test_image_question_full_answer_and_request_timing(tmp_path): + registry = Registry() + records = tmp_path / "requests.jsonl" + ModelRegistry._validate_ready({"event": "ready", "protocol_version": 2, + "capabilities": ["vision_language_generation"]}) + with client_for(registry, records) as client: + response = client.post("/v1/chat/completions", json=payload(registry), headers={"X-Request-ID": "vqa-42"}) + assert response.status_code == 200 and response.headers["x-request-id"] == "vqa-42" + body = response.json() + assert body["choices"][0]["message"] == {"role": "assistant", "content": registry.result["text"]} + assert body["usage"]["completion_tokens"] == 3 and body["object"] == "chat.completion" + record = json.loads(records.read_text()) + assert record["request_id"] == "vqa-42" and record["image_width"] == 8 and record["image_height"] == 4 + assert record["image_bytes"] == len(registry.image) and record["model_call_ms"] == 12.5 + assert record["timing_scope"] == "public_task_call_wall" and record["streaming"] == "none" + assert registry.closed == 1 and all(not path.exists() for path in registry.paths) + + +@pytest.mark.parametrize("kind", ["two_images", "two_turns", "system", "no_text", "stream", "stop", "thinking", "n", + "external_url", "local_path", "bad_media", "other_model", "tokens", "duplicate_tokens"]) +def test_unsupported_image_requests_never_invoke_native(kind): + registry = Registry() + body = payload(registry) + message = body["messages"][0] + if kind == "two_images": + message["content"].append(message["content"][-1]) + elif kind in {"system", "two_turns"}: + body["messages"].insert(0, {"role": "system" if kind == "system" else "user", "content": "Hi"}) + elif kind == "no_text": + message["content"].pop(0) + elif kind in {"external_url", "local_path", "bad_media"}: + message["content"][-1]["image_url"]["url"] = { + "external_url": "https://example.org/test.png", "local_path": "/etc/passwd", + "bad_media": "data:image/png;base64,???"}[kind] + else: + field, value = {"stream": ("stream", True), "stop": ("stop", "red"), "n": ("n", 2), + "thinking": ("enable_thinking", False), "other_model": ("model", "text-only"), + "tokens": ("max_tokens", 129), "duplicate_tokens": ("max_completion_tokens", 10)}[kind] + body[field] = value + with client_for(registry) as client: + response = client.post("/v1/chat/completions", json=body) + assert response.status_code == 400 and registry.calls == 0 + assert all(not path.exists() for path in registry.paths) + + +@pytest.mark.parametrize("text", ["", " "]) +def test_empty_native_text_is_preserved_for_the_family_to_grade(text): + registry = Registry() + registry.result.update(text=text, completion_tokens=0) + with client_for(registry) as client: + response = client.post("/v1/chat/completions", json=payload(registry)) + assert response.status_code == 200 + assert response.json()["choices"][0]["message"]["content"] == text + assert response.json()["usage"]["completion_tokens"] == 0 and not registry.retired + assert registry.closed == 1 and all(not path.exists() for path in registry.paths) + + +@pytest.mark.parametrize("patch", [{"image_width": 7}, {"image_height": True}, + {"timing_scope": "kernel"}, {"completion_tokens": True}, + {"model_call_ms": -1}]) +def test_invalid_native_result_retires_worker_and_releases_upload(patch): + registry = Registry() + registry.result.update(patch) + with client_for(registry) as client: + response = client.post("/v1/chat/completions", json=payload(registry)) + assert response.status_code == 502 and registry.retired and registry.closed == 1 + assert all(not path.exists() for path in registry.paths) + + +@pytest.mark.parametrize("failure,status", [("busy", 429), ("crashed", 503), ("pixels", 413)]) +def test_image_failure_records_and_cleanup(tmp_path, failure, status): + registry = Registry() + registry.failure = failure + records = tmp_path / "requests.jsonl" + with client_for(registry, records, max_image_pixels=16 if failure == "pixels" else 32) as client: + response = client.post("/v1/chat/completions", json=payload(registry)) + assert response.status_code == status and "private" not in response.text + record = json.loads(records.read_text()) + assert record["status"] == status and record["image_bytes"] == len(registry.image) + assert all(not path.exists() for path in registry.paths) diff --git a/apps/server/python/trtmc_server/app.py b/apps/server/python/trtmc_server/app.py index 1d5907adc8..36205f5842 100644 --- a/apps/server/python/trtmc_server/app.py +++ b/apps/server/python/trtmc_server/app.py @@ -32,7 +32,7 @@ WorkerTimeoutError, ) from .metrics import Metrics -from .protocol import chat_prompt, extract_embedding, extract_ranking, extract_result, extract_transcription, extract_detection, generation_config, public_worker_error +from .protocol import chat_prompt, chat_image, extract_vision_result, extract_embedding, extract_ranking, extract_result, extract_transcription, extract_detection, generation_config, public_worker_error from .records import RequestPolicy, RequestRecords from .registry import ModelRegistry from .schemas import ChatCompletionRequest, CompletionRequest, EmbeddingRequest, GenerationRequest, RankingRequest, TranscriptionRequest, DetectionRequest @@ -677,10 +677,67 @@ def validate(value): context["transcript_bytes"] = len(text.encode("utf-8")) return PlainTextResponse(text) if request.response_format == "text" else {"text": text} + async def vision_chat(http_request: Request, request: ChatCompletionRequest, + prompt: str, image_url: str) -> Any: + route = http_request.url.path + context = http_request.state.timing + context.update(model=request.model, input_items=1) + if (request.stream or request.stream_options is not None or request.n != 1 or + request.stop is not None or request.enable_thinking is not None): + metrics.reject(route, 400) + return error_response(400, "unsupported_parameter", + "image chat supports one nonstreaming answer without stop or thinking controls") + if not prompt.strip(): + metrics.reject(route, 400) + return error_response(400, "invalid_request", "image chat requires a text prompt", param="text") + if len(prompt.encode("utf-8")) > config.max_prompt_bytes: + metrics.reject(route, 413) + return error_response(413, "content_too_large", "prompt exceeds the server limit") + try: + default_tokens = registry.max_tokens(request.model, config.max_generation_tokens) + except ModelNotFoundError as error: + metrics.reject(route, 404) + return error_response(404, error.code, str(error), param="model") + max_tokens = request.max_completion_tokens or request.max_tokens or default_tokens + if max_tokens > config.max_generation_tokens: + metrics.reject(route, 400) + return error_response(400, "max_tokens_exceeded", "requested generation exceeds the server limit") + try: + data, suffix, metadata = await asyncio.to_thread(image_upload, image_url, config.max_image_pixels) + context.update(metadata) + except ImagePixelLimitError as error: + context.update(error.metadata) + metrics.reject(route, 413) + return error_response(413, "content_too_large", str(error)) + except ValueError as error: + metrics.reject(route, 400) + return error_response(400, "invalid_request", str(error)) + + worker_config = generation_config(request, max_tokens) + context["generation_config"] = worker_config + media = await create_uploaded_input(data, suffix) + + def validate(value): + return extract_vision_result(value, context["image_height"], context["image_width"]) + + result = await execute_task(http_request, request.model, "vision_language_generation", "generate_vision", + {"prompt": prompt, "image_path": str(media.path), "config": worker_config}, + validate, release_input=media.close) + if isinstance(result, JSONResponse): + return result + text, tokens, timings = validate(result) + context.update(timings, completion_tokens=tokens, completion_token_source="native_task") + return {"id": f"chatcmpl-{uuid.uuid4().hex}", "object": "chat.completion", + "created": int(time.time()), "model": request.model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, + "logprobs": None, "finish_reason": None}], + "usage": {"prompt_tokens": 0, "completion_tokens": tokens, "total_tokens": tokens}} + @app.post("/v1/chat/completions") async def chat_completions(http_request: Request, request: ChatCompletionRequest) -> Any: try: prompt, system_prompt = chat_prompt(request) + image_url = chat_image(request) except ValueError as error: metrics.reject("/v1/chat/completions", 400) return error_response(400, "invalid_request", str(error), param="messages") @@ -691,6 +748,8 @@ async def chat_completions(http_request: Request, request: ChatCompletionRequest "invalid_request", "max_tokens and max_completion_tokens are mutually exclusive", ) + if image_url is not None: + return await vision_chat(http_request, request, prompt, image_url) return await execute( "/v1/chat/completions", request, diff --git a/apps/server/python/trtmc_server/cli.py b/apps/server/python/trtmc_server/cli.py index 11293e5c1c..a7326bee5c 100644 --- a/apps/server/python/trtmc_server/cli.py +++ b/apps/server/python/trtmc_server/cli.py @@ -40,7 +40,7 @@ def positive_seconds(value: str) -> float: def parser() -> argparse.ArgumentParser: result = argparse.ArgumentParser( prog="trtmc-server", - description="Serve text-generation bundles alongside embedding, reranking, audio-transcription, and object-detection bundles through persistent native workers", + description="Serve text-generation bundles alongside vision-language, embedding, reranking, audio-transcription, and object-detection bundles through persistent native workers", ) result.add_argument("bundle", nargs="?", help="single .bundle artifact") result.add_argument("--model-name", help="API model name for the positional bundle") diff --git a/apps/server/python/trtmc_server/protocol.py b/apps/server/python/trtmc_server/protocol.py index 5a15243337..852f8f270b 100644 --- a/apps/server/python/trtmc_server/protocol.py +++ b/apps/server/python/trtmc_server/protocol.py @@ -10,7 +10,7 @@ from typing import Any from .errors import WorkerProtocolError, WorkerRemoteError, WorkerRequestTooLargeError -from .schemas import ChatCompletionRequest, GenerationRequest, TextContentPart +from .schemas import ChatCompletionRequest, GenerationRequest, TextContentPart, ImageContentPart def generation_config(request: GenerationRequest, max_tokens: int) -> dict[str, Any]: @@ -35,10 +35,30 @@ def chat_prompt(request: ChatCompletionRequest) -> tuple[str, str]: raise ValueError("messages must be one user message with an optional preceding system message") -def _message_text(content: str | list[TextContentPart]) -> str: +def chat_image(request: ChatCompletionRequest) -> str | None: + images = [part.image_url.url for message in request.messages if isinstance(message.content, list) + for part in message.content if isinstance(part, ImageContentPart)] + if not images: + return None + if len(images) != 1 or len(request.messages) != 1 or request.messages[0].role != "user": + raise ValueError("image chat requires one user message with exactly one image") + return images[0] + + +def _message_text(content: str | list[TextContentPart | ImageContentPart]) -> str: if isinstance(content, str): return content - return "".join(part.text for part in content) + return "".join(part.text for part in content if isinstance(part, TextContentPart)) + + +def extract_vision_result(result: Any, height: int, width: int) -> tuple[str, int, dict[str, float]]: + text, tokens, timings = extract_result(result) + if (type(result.get("image_height")) is not int or + type(result.get("image_width")) is not int or result["image_height"] != height or + result["image_width"] != width): + raise WorkerProtocolError("worker vision result has invalid text or image identity") + extract_task_timing(result) + return text, tokens, timings def extract_result(result: Any) -> tuple[str, int, dict[str, float]]: diff --git a/apps/server/python/trtmc_server/registry.py b/apps/server/python/trtmc_server/registry.py index 01aa388484..fa3f8c5ae0 100644 --- a/apps/server/python/trtmc_server/registry.py +++ b/apps/server/python/trtmc_server/registry.py @@ -179,5 +179,5 @@ def _validate_ready(metadata: dict[str, Any]) -> None: capabilities = metadata.get("capabilities") if (not isinstance(capabilities, list) or not capabilities or not all(isinstance(value, str) for value in capabilities) or - not {"text_generation", "embeddings", "reranking", "audio_transcription", "object_detection"}.intersection(capabilities)): + not {"text_generation", "embeddings", "reranking", "audio_transcription", "object_detection", "vision_language_generation"}.intersection(capabilities)): raise WorkerProtocolError("worker does not advertise a supported capability") diff --git a/apps/server/python/trtmc_server/schemas.py b/apps/server/python/trtmc_server/schemas.py index f362176128..ae49ca536c 100644 --- a/apps/server/python/trtmc_server/schemas.py +++ b/apps/server/python/trtmc_server/schemas.py @@ -19,9 +19,18 @@ class TextContentPart(StrictRequest): text: str +class ImageURL(StrictRequest): + url: str = Field(min_length=1) + + +class ImageContentPart(StrictRequest): + type: Literal["image_url"] + image_url: ImageURL + + class ChatMessage(StrictRequest): role: str - content: str | list[TextContentPart] + content: str | list[TextContentPart | ImageContentPart] class StreamOptions(StrictRequest): diff --git a/apps/server/tests/test_sdk_worker.cpp b/apps/server/tests/test_sdk_worker.cpp index 27e18bd105..ec61efc330 100644 --- a/apps/server/tests/test_sdk_worker.cpp +++ b/apps/server/tests/test_sdk_worker.cpp @@ -164,6 +164,19 @@ void test_task_selection_and_failures(const std::filesystem::path& root, run.messages[1].at("result").at("text") == item.second, "primary Task selection does not inject undeclared defaults or languages"); } + for (const auto& item : std::vector>{ + {"text_conversation", "language_fixture"}, {"images_text_to_text", "text_fixture"}}) { + std::istringstream input; + std::ostringstream output; + try { + trtmc::server::run_bundle_worker(write_bundle(root, item.first, item.second).string(), + options, input, output); + throw std::runtime_error("server selected a secondary Task instead of the primary"); + } catch (const trtmc::Error& error) { + require(error.code() == TRTMC_UNSUPPORTED && output.str().empty(), + "secondary capabilities cannot enable or replace a bundle's primary Task"); + } + } const auto failure = invoke(write_bundle(root, "must_not_run"), options, {generate("failure"), generate("must-not-retry")}); require(failure.status == 1 && failure.messages.size() == 2 && @@ -274,6 +287,36 @@ void test_detection_task(const std::filesystem::path& root, const trtmc::LoadOpt result.at("timing_scope") == "public_task_call_wall", "SDK defaults and complete native image detection results survive transport"); } + +void test_vision_task(const std::filesystem::path& root, const trtmc::LoadOptions& options, + const std::string& primary) { + const auto bundle = write_bundle(root, primary, "language_fixture"); + const auto path = root / "server-sdk-vision.png"; + std::vector pixels(8 * 4 * 3, 0.0F); + pixels[0] = 1.0F; + trtmc::cli::io::save_png(path.string(), pixels, 8, 4); + const Json valid{{"id", "vision"}, + {"op", "generate_vision"}, + {"image_path", path.string()}, + {"prompt", "Question?"}, + {"config", {{"suffix", "tail"}}}}; + auto invalid = valid; + invalid["config"]["unknown"] = 1; + const auto run = invoke(bundle, options, {invalid, valid}); + std::filesystem::remove(path); + require(run.status == 0 && run.messages.size() == 3 && + run.messages[0].at("capabilities") == Json::array({"vision_language_generation"}) && + run.messages[1].at("error").at("type") == "invalid_request_error", + "SDK vision capability and family-declared Config are honored"); + const auto& result = run.messages[2].at("result"); + const auto text = result.at("text").get(); + require(text.find("I:1;") != std::string::npos && + text.find("T:Question?;") != std::string::npos && text.size() >= 4 && + text.compare(text.size() - 4, 4, "tail") == 0 && result.at("image_height") == 4 && + result.at("image_width") == 8 && result.at("model_call_ms").get() >= 0 && + result.at("timing_scope") == "public_task_call_wall", + "actual public SDK Task receives image and question with its own Config"); +} } // namespace int main(int argc, char** argv) { @@ -290,6 +333,8 @@ int main(int argc, char** argv) { test_audio_task(root, options); test_audio_task_new_token_field(root, options); test_detection_task(root, options); + test_vision_task(root, options, "images_text_to_text"); + test_vision_task(root, options, "vision_language_generation"); return 0; } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/apps/server/tests/test_vision_worker.cpp b/apps/server/tests/test_vision_worker.cpp new file mode 100644 index 0000000000..ae02ce27c8 --- /dev/null +++ b/apps/server/tests/test_vision_worker.cpp @@ -0,0 +1,102 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "cli/io.h" +#include "server/native_worker.h" +#include "trtmc/task.h" + +#include +#include +#include +#include +#include +#include + +namespace { +using Json = nlohmann::json; +void require(bool condition, const char* message) { + if (!condition) + throw std::runtime_error(message); +} +class Vision final : public trtmc::IVisionLanguageGeneration, public trtmc::ITextGeneration { + public: + int calls{0}; + const char* task() const noexcept override { return IVisionLanguageGeneration::kTask; } + std::int32_t default_max_new_tokens() const override { return 10; } + trtmc::TextResult generate(const std::string&, const trtmc::TextGenerationConfig&) override { + throw std::runtime_error("image request must never take the text-only interface"); + } + trtmc::TextResult generate(const std::string& prompt, const float* pixels, std::int32_t height, + std::int32_t width, + const trtmc::TextGenerationConfig& config) override { + require(prompt == "Color?" && height == 4 && width == 8 && pixels[0] == 1.0F && + pixels[1] == 0.0F, + "original RGB geometry and question reach the Task"); + require(config.max_new_tokens == 10 && config.top_k == 1 && config.seed == 42, + "generation controls reach the owning family"); + ++calls; + trtmc::TextResult result; + result.text = "Red. Complete answer."; + result.token_ids = {1, 2, 3}; + return result; + } +}; +void verify(const std::string& path) { + Vision task; + const Json valid{{"id", "vqa"}, + {"op", "generate_vision"}, + {"prompt", "Color?"}, + {"image_path", path}, + {"config", {{"max_new_tokens", 10}, {"top_k", 1}, {"seed", 42}}}}; + std::stringstream input; + auto missing = valid; + missing.erase("image_path"); + input << missing.dump() << '\n'; + auto bad_config = valid; + bad_config["config"]["unknown"] = true; + input << bad_config.dump() << '\n'; + input << Json{{"id", "text"}, {"op", "generate"}, {"prompt", "Color?"}}.dump() << '\n'; + input << valid.dump() << '\n'; + auto default_config = valid; + default_config["id"] = "default-tokens"; + default_config["config"].erase("max_new_tokens"); + input << default_config.dump() << '\n'; + std::ostringstream output; + require(trtmc::server::run_task_worker(task, input, output) == 0, "worker completes"); + std::vector replies; + std::istringstream stream(output.str()); + for (std::string line; std::getline(stream, line);) + replies.push_back(Json::parse(line)); + require(replies.size() == 6 && task.calls == 2 && + replies[0].at("capabilities") == Json::array({"vision_language_generation"}) && + replies[0].at("default_max_new_tokens") == 10, + "declared bundle capability prevents inherited text dispatch"); + for (std::size_t i = 1; i < 4; ++i) + require(replies[i].at("error").at("type") == "invalid_request_error", + "invalid requests remain recoverable and never invoke the Task"); + const auto result = replies[4].at("result"); + require(result.at("text") == "Red. Complete answer." && result.at("completion_tokens") == 3 && + result.at("image_height") == 4 && result.at("image_width") == 8 && + result.at("model_call_ms").get() >= 0 && + result.at("timing_scope") == "public_task_call_wall", + "complete result and timing survive"); +} +} // namespace +int main() { + const auto path = std::filesystem::temp_directory_path() / + ("trtmc-vision-" + std::to_string(std::random_device{}()) + ".png"); + try { + std::vector pixels(8 * 4 * 3, 0.0F); + for (std::size_t i = 0; i < pixels.size(); i += 3) + pixels[i] = 1.0F; + trtmc::cli::io::save_png(path.string(), pixels, 8, 4); + verify(path.string()); + std::filesystem::remove(path); + } catch (const std::exception& error) { + std::filesystem::remove(path); + std::cerr << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/core/runtime/tensorrt/trt_module_impl.cpp b/core/runtime/tensorrt/trt_module_impl.cpp index bdd3a7d8d5..1b980ce425 100644 --- a/core/runtime/tensorrt/trt_module_impl.cpp +++ b/core/runtime/tensorrt/trt_module_impl.cpp @@ -257,7 +257,8 @@ void TrtModuleImpl::update_dynamic_shape(const std::string& name, BufferEntry& e dims.nbDims = static_cast(new_shape.size()); for (int32_t d = 0; d < dims.nbDims; ++d) dims.d[d] = new_shape[d]; - ctx_->setInputShape(name.c_str(), dims); + if (!ctx_->setInputShape(name.c_str(), dims)) + throw std::invalid_argument("TensorRT rejected dynamic shape for tensor '" + name + "'"); entry.shape = new_shape; } diff --git a/core/runtime/tests/test_trt_module_dynamic_input.cpp b/core/runtime/tests/test_trt_module_dynamic_input.cpp index ef28f4144f..ca9d41eb40 100644 --- a/core/runtime/tests/test_trt_module_dynamic_input.cpp +++ b/core/runtime/tests/test_trt_module_dynamic_input.cpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace { @@ -103,6 +104,33 @@ void test_backend_managed_graph_does_not_enable_nested_capture(nvinfer1::ICudaEn check(!module.cuda_graph_active(), "backend-managed graph leaves stream capture disabled"); } +void test_rejected_shape_fails_without_poisoning_the_next_request(nvinfer1::ICudaEngine& engine, + cudaStream_t stream) { + for (bool graph : {false, true}) { + trtmc::TrtModuleImpl module(&engine, engine.createExecutionContext(), stream); + if (graph) + module.enable_cuda_graph(); + float initial[] = {1.0F, 2.0F}; + (void)module.forward({{"input", trtmc::Tensor{initial, {2}, trtmc::DType::kFloat32}}}); + float oversized[] = {1.0F, 2.0F, 3.0F, 4.0F, 5.0F}; + bool rejected = false; + try { + (void)module.forward( + {{"input", trtmc::Tensor{oversized, {5}, trtmc::DType::kFloat32}}}); + } catch (const std::invalid_argument&) { + rejected = true; + } + check(rejected, "out-of-profile shapes must fail instead of returning stale-shape outputs"); + float next[] = {7.0F, 8.0F}; + const auto output = + module.forward({{"input", trtmc::Tensor{next, {2}, trtmc::DType::kFloat32}}}); + const auto& tensor = output.at("output"); + const auto* values = static_cast(tensor.data); + check(tensor.numel() == 2 && values[0] == 7.0F && values[1] == 8.0F, + "a rejected shape leaves the next valid request usable"); + } +} + void test_short_static_upload_does_not_keep_the_previous_input(nvinfer1::ICudaEngine& engine, cudaStream_t stream) { for (bool graph : {false, true}) { @@ -162,6 +190,7 @@ int main() { test_first_forward_allocates_unbound_dynamic_input(*engine, stream); test_external_dynamic_input_stays_external(*engine, stream); test_backend_managed_graph_does_not_enable_nested_capture(*engine, stream); + test_rejected_shape_fails_without_poisoning_the_next_request(*engine, stream); test_short_static_upload_does_not_keep_the_previous_input(*static_engine, stream); test_short_external_static_upload_preserves_its_tail(*static_engine, stream); cudaStreamDestroy(stream); diff --git a/families/deepseek_ocr/model.py b/families/deepseek_ocr/model.py index c5eed208af..87a67a271e 100644 --- a/families/deepseek_ocr/model.py +++ b/families/deepseek_ocr/model.py @@ -679,7 +679,7 @@ def get_vl_config(self, config: ModelConfig) -> dict | None: "num_image_pad_tokens": 145, # 144 projected + 1 view_separator "prefill_max_length": MAX_SEQUENCE_PREFILL_LENGTH, "vision_output_dim": hidden, - "preprocessor_type": "simple_chw", + "preprocessor_type": "pad_center_chw", "image_mean": [0.5, 0.5, 0.5], "image_std": [0.5, 0.5, 0.5], "interpolation": "bicubic", diff --git a/families/deepseek_ocr/runtime/CMakeLists.txt b/families/deepseek_ocr/runtime/CMakeLists.txt index d496d77d03..063d09ee2f 100644 --- a/families/deepseek_ocr/runtime/CMakeLists.txt +++ b/families/deepseek_ocr/runtime/CMakeLists.txt @@ -49,6 +49,21 @@ install(TARGETS trtmc_model_deepseek_ocr ) if(TRTMC_BUILD_TESTS) + add_executable(test_deepseek_ocr_image_padding + ../tests/cpp/test_image_padding.cpp + ) + target_include_directories(test_deepseek_ocr_image_padding PRIVATE + ${PROJECT_SOURCE_DIR} + ) + target_link_libraries(test_deepseek_ocr_image_padding PRIVATE + trtmc_model_deepseek_ocr + ) + target_compile_options(test_deepseek_ocr_image_padding PRIVATE + -Wall -Wextra -Wpedantic + ) + add_test(NAME deepseek_ocr_image_padding COMMAND test_deepseek_ocr_image_padding) + set_tests_properties(deepseek_ocr_image_padding PROPERTIES LABELS cpu) + add_executable(test_deepseek_ocr_runtime_config ../tests/cpp/test_runtime_config.cpp ) diff --git a/families/deepseek_ocr/runtime/image_preprocessor.cpp b/families/deepseek_ocr/runtime/image_preprocessor.cpp index 8ba7fee487..6f0b79fe55 100644 --- a/families/deepseek_ocr/runtime/image_preprocessor.cpp +++ b/families/deepseek_ocr/runtime/image_preprocessor.cpp @@ -11,6 +11,7 @@ #include "stb_image_resize2.h" #include +#include #include #include #include @@ -43,6 +44,13 @@ struct LoadedImage { bool ok{false}; }; +// PIL uses Python's ties-to-even rounding for contain and center placement. +static int round_even(double value) { + const auto lower = static_cast(std::floor(value)); + const auto fraction = value - lower; + return lower + (fraction > 0.5 || (fraction == 0.5 && lower % 2 != 0)); +} + // Resize raw uint8 RGB buffer to target_size x target_size using the given filter. static std::vector resize_raw(const unsigned char* raw, int width, int height, int target_size, stbir_filter filter) { @@ -220,13 +228,14 @@ load_pad_center_resize_normalize(const runtime::adapters::io::DecodedImage& imag const int target_size = config.fixed_image_size; const stbir_filter filter = resolve_stbir_filter(config.interpolation); - // Compute scaled dimensions that fit inside target_size x target_size - // This matches PIL ImageOps.pad behavior: scale to fit, then center. - const float scale_w = static_cast(target_size) / static_cast(image.width); - const float scale_h = static_cast(target_size) / static_cast(image.height); - const float scale = std::min(scale_w, scale_h); - const int new_w = std::max(1, static_cast(image.width * scale)); - const int new_h = std::max(1, static_cast(image.height * scale)); + int new_w = target_size; + int new_h = target_size; + if (image.width > image.height) + new_h = + std::max(1, round_even(static_cast(image.height) / image.width * target_size)); + else if (image.height > image.width) + new_w = + std::max(1, round_even(static_cast(image.width) / image.height * target_size)); // Resize preserving aspect ratio std::vector resized_small(static_cast(new_w) * new_h * 3); @@ -253,8 +262,8 @@ load_pad_center_resize_normalize(const runtime::adapters::io::DecodedImage& imag } // Center the resized image in the padded canvas - const int x_off = (target_size - new_w) / 2; - const int y_off = (target_size - new_h) / 2; + const int x_off = round_even((target_size - new_w) * 0.5); + const int y_off = round_even((target_size - new_h) * 0.5); for (int y = 0; y < new_h; ++y) { const unsigned char* src_row = resized_small.data() + static_cast(y) * new_w * 3; diff --git a/families/deepseek_ocr/tests/cpp/test_image_padding.cpp b/families/deepseek_ocr/tests/cpp/test_image_padding.cpp new file mode 100644 index 0000000000..ea7654cc35 --- /dev/null +++ b/families/deepseek_ocr/tests/cpp/test_image_padding.cpp @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "families/deepseek_ocr/runtime/image_preprocessor.h" + +#include +#include +#include + +namespace { +void verify(int width, int height, int left, int top, int right, int bottom) { + trtmc::runtime::adapters::io::DecodedImage image; + image.width = width; + image.height = height; + image.channels = 3; + image.pixels.assign(width * height * 3, 255); + trtmc::DeepseekOcrPreprocessConfig config; + config.fixed_image_size = 16; + config.preprocessor_type = "pad_center_chw"; + for (int channel = 0; channel < 3; ++channel) { + config.image_mean[channel] = 0.5F; + config.image_std[channel] = 0.5F; + } + const auto result = trtmc::deepseek_ocr_preprocess_decoded_image(image, config); + if (!result.ok || result.pixel_values.size() != 3 * 16 * 16) + throw std::runtime_error("padding must produce the declared CHW image"); + for (int channel = 0; channel < 3; ++channel) { + for (int y = 0; y < 16; ++y) { + for (int x = 0; x < 16; ++x) { + const bool content = x >= left && x < right && y >= top && y < bottom; + const float expected = content ? 1.0F : -1.0F / 255; + const auto value = result.pixel_values[channel * 256 + y * 16 + x]; + if (std::abs(value - expected) > 1e-6F) + throw std::runtime_error("geometry or mean-color padding differs from PIL"); + } + } + } +} +} // namespace + +int main() { + try { + // Independent PIL ImageOps.pad white-image bounding boxes at size16. + verify(3, 7, 4, 0, 11, 16); + verify(7, 3, 0, 4, 16, 11); + verify(13, 16, 2, 0, 15, 16); + verify(16, 13, 0, 2, 16, 15); + verify(16, 16, 0, 0, 16, 16); + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/families/deepseek_ocr/tests/test_builder_contract.py b/families/deepseek_ocr/tests/test_builder_contract.py index 144178741f..aa21997800 100644 --- a/families/deepseek_ocr/tests/test_builder_contract.py +++ b/families/deepseek_ocr/tests/test_builder_contract.py @@ -325,14 +325,14 @@ def test_deepseek_ocr_resizes_sam_position_embedding_for_768_view() -> None: np.testing.assert_allclose(resized, 1.0, atol=1e-6) -def test_deepseek_ocr_uses_official_768_single_view_contract() -> None: +def test_deepseek_ocr_uses_bounded_768_aspect_preserving_view() -> None: vl_config = model._DeepseekOcrModel().get_vl_config(_config()) assert vl_config is not None assert vl_config["fixed_image_size"] == 768 assert vl_config["num_image_pad_tokens"] == 145 assert vl_config["prefill_max_length"] == 256 - assert vl_config["preprocessor_type"] == "simple_chw" + assert vl_config["preprocessor_type"] == "pad_center_chw" @pytest.mark.parametrize( diff --git a/families/deepseek_ocr/tests/test_vision_oracle.py b/families/deepseek_ocr/tests/test_vision_oracle.py index 342f424132..67690ace2b 100644 --- a/families/deepseek_ocr/tests/test_vision_oracle.py +++ b/families/deepseek_ocr/tests/test_vision_oracle.py @@ -56,6 +56,7 @@ def test_native_pixels_follow_the_family_runtime_contract() -> None: image, { "fixed_image_size": 4, + "preprocessor_type": "simple_chw", "image_mean": [0.5, 0.5, 0.5], "image_std": [0.5, 0.5, 0.5], }, @@ -71,6 +72,17 @@ def test_vision_gate_is_the_active_half_cosine_contract() -> None: assert_vision_parity([[1.0, 0.0]], [[-1.0, 0.0]]) +def test_padded_pixels_keep_white_text_area_and_mean_color_border() -> None: + pixels = _native_pixels(Image.new("RGB", (13, 16), "white"), { + "fixed_image_size": 16, "preprocessor_type": "pad_center_chw", + "image_mean": [0.5] * 3, "image_std": [0.5] * 3, + }) + assert pixels.shape == (1, 3, 16, 16) + np.testing.assert_allclose(pixels[:, :, :, 2:15], 1.0) + np.testing.assert_allclose(pixels[:, :, :, :2], -1 / 255, atol=1e-7) + np.testing.assert_allclose(pixels[:, :, :, 15:], -1 / 255, atol=1e-7) + + @pytest.mark.parametrize("features", [[0.0, 0.0], [float("inf"), 1.0]]) def test_vision_gate_rejects_invalid_native_features(features) -> None: with pytest.raises(AssertionError): diff --git a/families/deepseek_ocr/tests/vision_oracle.py b/families/deepseek_ocr/tests/vision_oracle.py index de4db4d898..ecb5655307 100644 --- a/families/deepseek_ocr/tests/vision_oracle.py +++ b/families/deepseek_ocr/tests/vision_oracle.py @@ -28,10 +28,16 @@ def _bundle_section(bundle: Path, name: str) -> bytes: def _native_pixels(image, config: dict) -> np.ndarray: - from PIL import Image + from PIL import Image, ImageOps size = int(config["fixed_image_size"]) - resized = image.convert("RGB").resize((size, size), Image.Resampling.BICUBIC) + image = image.convert("RGB") + if config["preprocessor_type"] == "pad_center_chw": + color = tuple(int(value * 255) for value in config["image_mean"]) + resized = ImageOps.pad(image, (size, size), color=color) + else: + assert config["preprocessor_type"] == "simple_chw" + resized = image.resize((size, size), Image.Resampling.BICUBIC) pixels = np.asarray(resized, dtype=np.float32) / 255.0 mean = np.asarray(config["image_mean"], dtype=np.float32) std = np.asarray(config["image_std"], dtype=np.float32) diff --git a/families/internvl/model.py b/families/internvl/model.py index d8c899bab9..f98cf0f6f4 100644 --- a/families/internvl/model.py +++ b/families/internvl/model.py @@ -226,11 +226,11 @@ def _load_internvl_text_weights( model_dir: str, config: ModelConfig, ) -> WeightDict: - """Load InternVL3 text decoder weights. + """Load InternVL text decoder weights. InternVL3-8B-hf uses model.language_model.model.layers.{i}.* prefix. Falls back to model.layers.{i}.* if language_model prefix not found. - The text decoder is standard Qwen2 architecture. + Qwen2 and Qwen3 decoder blocks share the graph; Qwen3 adds per-head Q/K norms. """ from pathlib import Path @@ -313,6 +313,13 @@ def _load_internvl_text_weights( raw = _load_tensor(readers, full_key).astype(np.float32) weights[f"{prefix}.{proj_name}"] = raw + # Qwen3 normalizes each projected Q/K head before RoPE. The graph + # already supports these optional weights; preserve them at loading. + for name in ("q_norm", "k_norm"): + key = f"{hf_prefix}.self_attn.{name}.weight" + if _has_tensor(readers, key): + weights[f"{prefix}.{name}"] = _load_tensor(readers, key).astype(np.float32) + # SwiGLU MLP gate_raw = _load_tensor(readers, f"{hf_prefix}.mlp.gate_proj.weight") up_raw = _load_tensor(readers, f"{hf_prefix}.mlp.up_proj.weight") @@ -463,6 +470,7 @@ def _build_native(request: "BuildRequest", writer: "BundleWriter") -> None: parallel.validate() config.raw["_model_dir"] = str(model_dir) model = _InternVLModel() + model_vl = model.get_vl_config(config) or {} weights = model.load_weights(str(model_dir), config) writer.set_header(family="internvl", task=request.task, backend=request.backend) if parallel.enabled: @@ -490,6 +498,9 @@ def _build_native(request: "BuildRequest", writer: "BundleWriter") -> None: verbose=request.verbose, parallel_config=parallel, ) + # Stage serialized plans on disk before building the next one. + writer.add_bytes("prefill.plan", prefill) + del prefill config.raw["_decoder_engine_role"] = "decode" decode = model.build_engine( config, @@ -502,13 +513,15 @@ def _build_native(request: "BuildRequest", writer: "BundleWriter") -> None: ) config.raw.pop("_decoder_engine_role", None) writer.add_bytes("engine.plan", decode) - writer.add_bytes("prefill.plan", prefill) + del decode vision = model.build_vision_engine( str(model_dir), config, weights, precision=precision, verbose=request.verbose ) if vision is None: raise RuntimeError("InternVL vision build returned no engine") - vl = model.get_vl_config(config) or {} + writer.add_bytes("vision.plan", vision) + del vision + vl = model_vl runtime = { "tensor_parallel_size": parallel.tp_size, "num_layers": config.num_hidden_layers, @@ -532,7 +545,6 @@ def _build_native(request: "BuildRequest", writer: "BundleWriter") -> None: # token cache; the tensor-parallel decoder keeps one tile until its # single-query attention avoids masked IAttention over >512 rows. runtime.update({"preprocessor_type": "simple_chw", "max_image_tiles": 1}) - writer.add_bytes("vision.plan", vision) runtime.update(_tokenizer_runtime_contract(model_dir)) writer.add_json("runtime.json", runtime) for filename in ( diff --git a/families/internvl/tests/cpp/test_image_normalization.cpp b/families/internvl/tests/cpp/test_image_normalization.cpp index adfb1f8477..fd6135211c 100644 --- a/families/internvl/tests/cpp/test_image_normalization.cpp +++ b/families/internvl/tests/cpp/test_image_normalization.cpp @@ -82,12 +82,52 @@ void check_uniform_image(const trtmc::InternVlPreprocessConfig& config, std::uin check(seen[0] && seen[1] && seen[2], label + " pixels cover every channel"); } +void check_dynamic_tiles(bool portrait) { + auto document = nlohmann::json::parse(runtime_json()); + document["preprocessor_type"] = "dynamic_tiles_chw"; + document["max_image_tiles"] = 2; + document["fixed_image_size"] = 4; + document["image_mean"] = std::array{0, 0, 0}; + document["image_std"] = std::array{1, 1, 1}; + const auto config = trtmc::internvl_parse_preprocess_config(document.dump()); + trtmc::runtime::adapters::io::DecodedImage image; + image.width = portrait ? 4 : 8; + image.height = portrait ? 8 : 4; + image.channels = 3; + image.pixels.assign(image.width * image.height * 3, 0); + for (int y = 0; y < image.height; ++y) + for (int x = 0; x < image.width; ++x) { + const bool first = portrait ? y < 4 : x < 4; + image.pixels[(y * image.width + x) * 3 + (first ? 0 : 2)] = 255; + } + const auto result = trtmc::internvl_preprocess_decoded_image(image, config); + check(result.ok && result.num_tiles == 3 && result.pixel_values.size() == 144, + "two local tiles plus one thumbnail"); + if (result.pixel_values.size() != 144) + return; + for (int pixel = 0; pixel < 16; ++pixel) { + check(result.pixel_values[pixel] == 1 && result.pixel_values[32 + pixel] == 0, + "first tile is red in row-major order"); + check(result.pixel_values[48 + pixel] == 0 && result.pixel_values[80 + pixel] == 1, + "second tile is blue in row-major order"); + } + check(result.pixel_values[96] > result.pixel_values[111], + "thumbnail follows local tiles and preserves the complete image"); + auto prompt_config = config; + prompt_config.num_image_pad_tokens *= result.num_tiles; + const auto prompt = trtmc::internvl_format_prompt("Question?", prompt_config); + check(prompt.size() == 3 * 256 * config.image_token_str.size() + 9, + "prompt reserves every tile's image tokens"); +} + } // namespace int main() { const auto config = trtmc::internvl_parse_preprocess_config(runtime_json()); check_uniform_image(config, 0); check_uniform_image(config, 255); + check_dynamic_tiles(false); + check_dynamic_tiles(true); if (g_failures != 0) { std::cerr << g_failures << " InternVL image normalization check(s) failed\n"; return 1; diff --git a/families/internvl/tests/manifests/internvl3-14b.json b/families/internvl/tests/manifests/internvl3-14b.json new file mode 100644 index 0000000000..d722d40d21 --- /dev/null +++ b/families/internvl/tests/manifests/internvl3-14b.json @@ -0,0 +1,26 @@ +{ + "name": "internvl3-14b", + "hf_id": "OpenGVLab/InternVL3-14B-hf", + "hf_revision": "e22931943e5336f85e06f4e2b38f3e5e6ee4de3b", + "bundle": "internvl3-14b-vl.bundle", + "family": "internvl", + "task": "vision_language_generation", + "precision": "fp16", + "testcases": [ + { + "name": "internvl3-14b", + "premerge": true, + "reference_precision": "fp32", + "reference_backend": "hf_transformers", + "prompt": "What color is the vehicle in this image? Answer in one word.", + "max_new_tokens": 10, + "test_image": "data/test_img.jpeg", + "community_gpu": false, + "metadata": { + "community_gpu_deferred_reason": "The standard Community GPU has 24 GB VRAM; this TP1 profile holds two decoder engines that exceed that capacity. The full premerge testcase remains enabled for capable validation hosts." + } + } + ], + "tensor_parallel_size": 1, + "max_sequence_length": 4096 +} diff --git a/families/internvl/tests/manifests/internvl3-1b.json b/families/internvl/tests/manifests/internvl3-1b.json new file mode 100644 index 0000000000..e169f612ad --- /dev/null +++ b/families/internvl/tests/manifests/internvl3-1b.json @@ -0,0 +1,22 @@ +{ + "name": "internvl3-1b", + "hf_id": "OpenGVLab/InternVL3-1B-hf", + "hf_revision": "014c0583a0d4bedf29fbe2dbff4f865eb998e171", + "bundle": "internvl3-1b-vl.bundle", + "family": "internvl", + "task": "vision_language_generation", + "precision": "fp16", + "testcases": [ + { + "name": "internvl3-1b", + "premerge": true, + "reference_precision": "fp32", + "reference_backend": "hf_transformers", + "prompt": "What color is the vehicle in this image? Answer in one word.", + "max_new_tokens": 10, + "test_image": "data/test_img.jpeg" + } + ], + "tensor_parallel_size": 1, + "max_sequence_length": 4096 +} diff --git a/families/internvl/tests/manifests/internvl35-14b.json b/families/internvl/tests/manifests/internvl35-14b.json new file mode 100644 index 0000000000..364f500869 --- /dev/null +++ b/families/internvl/tests/manifests/internvl35-14b.json @@ -0,0 +1,26 @@ +{ + "name": "internvl35-14b", + "hf_id": "OpenGVLab/InternVL3_5-14B-HF", + "hf_revision": "226b96d5912e69159abc0384cefcbd51487fdce0", + "bundle": "internvl35-14b-vl.bundle", + "family": "internvl", + "task": "vision_language_generation", + "precision": "fp16", + "testcases": [ + { + "name": "internvl35-14b", + "premerge": true, + "reference_precision": "fp32", + "reference_backend": "hf_transformers", + "prompt": "What color is the vehicle in this image? Answer in one word.", + "max_new_tokens": 10, + "test_image": "data/test_img.jpeg", + "community_gpu": false, + "metadata": { + "community_gpu_deferred_reason": "The standard Community GPU has 24 GB VRAM; this TP1 profile holds two decoder engines that exceed that capacity. The full premerge testcase remains enabled for capable validation hosts." + } + } + ], + "tensor_parallel_size": 1, + "max_sequence_length": 4096 +} diff --git a/families/internvl/tests/manifests/internvl35-1b.json b/families/internvl/tests/manifests/internvl35-1b.json new file mode 100644 index 0000000000..3d83aa051d --- /dev/null +++ b/families/internvl/tests/manifests/internvl35-1b.json @@ -0,0 +1,22 @@ +{ + "name": "internvl35-1b", + "hf_id": "OpenGVLab/InternVL3_5-1B-HF", + "hf_revision": "9191dbccf312b537016f041b25d61c72e7c5c9f3", + "bundle": "internvl35-1b-vl.bundle", + "family": "internvl", + "task": "vision_language_generation", + "precision": "fp16", + "testcases": [ + { + "name": "internvl35-1b", + "premerge": true, + "reference_precision": "fp32", + "reference_backend": "hf_transformers", + "prompt": "What color is the vehicle in this image? Answer in one word.", + "max_new_tokens": 10, + "test_image": "data/test_img.jpeg" + } + ], + "tensor_parallel_size": 1, + "max_sequence_length": 4096 +} diff --git a/families/internvl/tests/manifests/internvl35-2b.json b/families/internvl/tests/manifests/internvl35-2b.json new file mode 100644 index 0000000000..4c1979f07b --- /dev/null +++ b/families/internvl/tests/manifests/internvl35-2b.json @@ -0,0 +1,22 @@ +{ + "name": "internvl35-2b", + "hf_id": "OpenGVLab/InternVL3_5-2B-HF", + "hf_revision": "3f301ffcf3dcbb47893afae6650ea3e78d96fb6d", + "bundle": "internvl35-2b-vl.bundle", + "family": "internvl", + "task": "vision_language_generation", + "precision": "fp16", + "testcases": [ + { + "name": "internvl35-2b", + "premerge": true, + "reference_precision": "fp32", + "reference_backend": "hf_transformers", + "prompt": "What color is the vehicle in this image? Answer in one word.", + "max_new_tokens": 10, + "test_image": "data/test_img.jpeg" + } + ], + "tensor_parallel_size": 1, + "max_sequence_length": 4096 +} diff --git a/families/internvl/tests/manifests/internvl35-4b.json b/families/internvl/tests/manifests/internvl35-4b.json new file mode 100644 index 0000000000..e5518116f8 --- /dev/null +++ b/families/internvl/tests/manifests/internvl35-4b.json @@ -0,0 +1,22 @@ +{ + "name": "internvl35-4b", + "hf_id": "OpenGVLab/InternVL3_5-4B-HF", + "hf_revision": "6bd4487402110ef9889ba50eb7aefeb302526fed", + "bundle": "internvl35-4b-vl.bundle", + "family": "internvl", + "task": "vision_language_generation", + "precision": "fp16", + "testcases": [ + { + "name": "internvl35-4b", + "premerge": true, + "reference_precision": "fp32", + "reference_backend": "hf_transformers", + "prompt": "What color is the vehicle in this image? Answer in one word.", + "max_new_tokens": 10, + "test_image": "data/test_img.jpeg" + } + ], + "tensor_parallel_size": 1, + "max_sequence_length": 4096 +} diff --git a/families/internvl/tests/manifests/internvl35-8b.json b/families/internvl/tests/manifests/internvl35-8b.json new file mode 100644 index 0000000000..58002ec840 --- /dev/null +++ b/families/internvl/tests/manifests/internvl35-8b.json @@ -0,0 +1,26 @@ +{ + "name": "internvl35-8b", + "hf_id": "OpenGVLab/InternVL3_5-8B-HF", + "hf_revision": "741a7d03020411e666c6109218ab71e08151ef86", + "bundle": "internvl35-8b-vl.bundle", + "family": "internvl", + "task": "vision_language_generation", + "precision": "fp16", + "testcases": [ + { + "name": "internvl35-8b", + "premerge": true, + "reference_precision": "fp32", + "reference_backend": "hf_transformers", + "prompt": "What color is the vehicle in this image? Answer in one word.", + "max_new_tokens": 10, + "test_image": "data/test_img.jpeg", + "community_gpu": false, + "metadata": { + "community_gpu_deferred_reason": "The standard Community GPU has 24 GB VRAM; this TP1 profile holds two decoder engines that exceed that capacity. The full premerge testcase remains enabled for capable validation hosts." + } + } + ], + "tensor_parallel_size": 1, + "max_sequence_length": 4096 +} diff --git a/families/internvl/tests/test_e2e.py b/families/internvl/tests/test_e2e.py index a84e18b678..8398ee71e9 100644 --- a/families/internvl/tests/test_e2e.py +++ b/families/internvl/tests/test_e2e.py @@ -31,6 +31,15 @@ def _case_index() -> dict[str, tuple[Path, dict, dict]]: assert manifest["family"] == FAMILY assert manifest["task"] in TASKS for case in manifest["testcases"]: + community = case.get("community_gpu", True) + assert type(community) is bool, "community_gpu must be boolean" + if not community: + # This declares the public runner's capacity only. Explicit + # family E2E selectors still execute the complete testcase. + reason = case.get("metadata", {}).get("community_gpu_deferred_reason") + assert isinstance(reason, str) and reason.strip(), ( + "a Community capacity deferral requires an owning reason" + ) name = str(case["name"]) assert name not in result result[name] = (path, manifest, case) @@ -129,23 +138,22 @@ def _runtime(manifest: dict) -> tuple[Path, Path]: def _build(model_dir: Path, bundle: Path, manifest: dict) -> None: - build( - BuildRequest( - model_dir=model_dir, - output_path=bundle, - family=FAMILY, - task=manifest["task"], - precision=manifest["precision"], - max_sequence_length=manifest.get("max_sequence_length"), - image_height=manifest.get("image_height"), - image_width=manifest.get("image_width"), - video_num_frames=manifest.get("video_num_frames"), - max_batch_size=int(manifest.get("max_batch_size", 1)), - tensor_parallel_size=int(manifest["tensor_parallel_size"]), - quantization=manifest.get("quantization"), - fp32_layers=tuple((int(layer) for layer in manifest.get("fp32_layers", ()))), - ) + request = BuildRequest( + model_dir=model_dir, + output_path=bundle, + family=FAMILY, + task=manifest["task"], + precision=manifest["precision"], + max_sequence_length=manifest.get("max_sequence_length"), + image_height=manifest.get("image_height"), + image_width=manifest.get("image_width"), + video_num_frames=manifest.get("video_num_frames"), + max_batch_size=int(manifest.get("max_batch_size", 1)), + tensor_parallel_size=int(manifest["tensor_parallel_size"]), + quantization=manifest.get("quantization"), + fp32_layers=tuple((int(layer) for layer in manifest.get("fp32_layers", ()))), ) + build(request) def _run_json( @@ -455,6 +463,13 @@ def test_2b_comparator_accepts_ned_or_word_agreement() -> None: def test_reference_routes_keep_8b_on_golden_and_2b_on_hf() -> None: expected_backends = { + "internvl3-1b": "hf_transformers", + "internvl35-1b": "hf_transformers", + "internvl35-2b": "hf_transformers", + "internvl35-4b": "hf_transformers", + "internvl35-8b": "hf_transformers", + "internvl35-14b": "hf_transformers", + "internvl3-14b": "hf_transformers", "internvl3-2b": "hf_transformers", "internvl3-2b-tp2": "hf_transformers", "internvl3-8b": "golden_snapshot", diff --git a/families/internvl/tests/test_plan_lifetime.py b/families/internvl/tests/test_plan_lifetime.py new file mode 100644 index 0000000000..a9e0fa048c --- /dev/null +++ b/families/internvl/tests/test_plan_lifetime.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep serialized engines out of RAM while building the next network.""" + +import gc +import json +from types import SimpleNamespace + +import pytest + + +def test_each_plan_is_staged_and_released_before_the_next_build(tmp_path, monkeypatch): + pytest.importorskip("tensorrt") + from families.internvl import model + from families.internvl.edge_llm import dispatch + from tensorrt_model_connect.bundle_writer import BundleWriter + + live = set() + built = [] + + class Plan(bytes): + def __new__(cls, role): + value = super().__new__(cls, role.encode()) + value.role = role + live.add(role) + return value + + def __del__(self): + live.remove(self.role) + + def next_plan(role): + gc.collect() + assert not live, f"previous serialized engine retained during {role}: {live}" + built.append(role) + return Plan(role) + + class Model: + @staticmethod + def load_weights(*_args): + return {} + + @staticmethod + def build_engine(config, *_args, **_kwargs): + return next_plan(config.raw["_decoder_engine_role"]) + + @staticmethod + def build_vision_engine(*_args, **_kwargs): + return next_plan("vision") + + @staticmethod + def get_vl_config(_config): + return {"image_token_id": 3, "vision_output_dim": 4} + + monkeypatch.setattr(model, "_InternVLModel", Model) + monkeypatch.setattr(model, "_tokenizer_runtime_contract", lambda _directory: {}) + monkeypatch.setattr(dispatch, "build", lambda request, writer, native: native(request, writer)) + (tmp_path / "config.json").write_text(json.dumps({ + "model_type": "internvl", "hidden_size": 4, "num_hidden_layers": 1, + "vocab_size": 8, "num_attention_heads": 1, "num_key_value_heads": 1, + })) + request = SimpleNamespace( + mtp_seq_len=None, dynamic_kv_cache=False, image_height=None, image_width=None, + video_num_frames=None, max_batch_size=1, context_parallel_size=1, + task="vision_language_generation", tensor_parallel_size=1, quantization=None, + fp32_layers=(), model_dir=tmp_path, precision="fp16", max_sequence_length=768, + verbose=False, backend="trt", family="internvl", + output_path=tmp_path / "model.bundle", graph_transform=None, + ) + writer = BundleWriter(tmp_path / "model.bundle") + try: + model.build(request, writer) + writer.finish() + finally: + writer.abort() + gc.collect() + assert built == ["prefill", "decode", "vision"] + assert not live + assert (tmp_path / "model.bundle").is_file() diff --git a/families/internvl/tests/test_prompt_framing.py b/families/internvl/tests/test_prompt_framing.py new file mode 100644 index 0000000000..5192a38963 --- /dev/null +++ b/families/internvl/tests/test_prompt_framing.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compare one-view native framing with actual upstream media insertion.""" + +from types import SimpleNamespace + +import numpy as np +import pytest + +pytest.importorskip("tensorrt") +pytest.importorskip("transformers.models.internvl.processing_internvl") + +from transformers import InternVLProcessor + +from families.internvl.model import _InternVLModel + + +def test_native_prompt_matches_upstream_media_insertion() -> None: + processor = SimpleNamespace( + start_image_token="", end_image_token="", + image_token="", video_token="