diff --git a/CMakeLists.txt b/CMakeLists.txt index e4e60c61d8..185761ff78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -552,6 +552,13 @@ if(TRTMC_BUILD_TESTS) target_compile_options(test_server_detection_worker PRIVATE -Wall -Wextra -Wpedantic) add_test(NAME server_detection_worker COMMAND test_server_detection_worker) set_tests_properties(server_detection_worker PROPERTIES LABELS cpu) + add_executable(test_server_vision_worker apps/server/tests/test_vision_worker.cpp) + target_include_directories(test_server_vision_worker PRIVATE + ${PROJECT_SOURCE_DIR}/core/runtime/include ${PROJECT_SOURCE_DIR}/apps) + target_link_libraries(test_server_vision_worker PRIVATE trtmc_server trtmc_media_io nlohmann_json::nlohmann_json) + target_compile_options(test_server_vision_worker PRIVATE -Wall -Wextra -Wpedantic) + add_test(NAME server_vision_worker COMMAND test_server_vision_worker) + set_tests_properties(server_vision_worker PROPERTIES LABELS cpu) endif() add_library(trtmc_test_cli_fixture SHARED apps/cli/tests/test_family_cli.cpp) @@ -725,6 +732,7 @@ if(TRTMC_BUILD_TESTS) add_dependencies(test_server_sdk_worker trtmc-server trtmc_api_test_backend trtmc_api_test_family trtmc_api_test_text_family trtmc_api_test_stream_family trtmc_api_test_audio_family trtmc_api_test_image_boxes_family) + add_dependencies(test_server_sdk_worker trtmc_api_test_language_family) add_test(NAME server_sdk_worker COMMAND test_server_sdk_worker "${_trtmc_api_test_runtime_root}") add_test(NAME server_sdk_worker_process diff --git a/apps/aiperf_qual/README.md b/apps/aiperf_qual/README.md index d88e6c934a..6705d0b9f9 100644 --- a/apps/aiperf_qual/README.md +++ b/apps/aiperf_qual/README.md @@ -404,7 +404,7 @@ PYTHONPATH=apps/aiperf_qual:apps/benchmark:core/builder:. \ python -m trtmc_aiperf_qual endpoint-models --repo . --group embeddings ``` -Groups are `embeddings`, `reranking`, `transcription`, and `detection`. +Groups are `embeddings`, `reranking`, `transcription`, `detection`, and `vision`. The inventory selects a normal full-model profile when one is available, with precision and parallel variants listed separately. Checkpoints without one remain explicit gaps. Discovery reports `unverified`; it does not claim @@ -437,10 +437,22 @@ PYTHONPATH=apps/aiperf_qual:apps/benchmark:apps/perf_serving:core/builder:. \ --out /path/to/new-result-directory ``` +The default client tokenizer uses the profile's pinned checkpoint revision. +Use `--tokenizer /path/to/prepared-tokenizer` when its files live separately; +`--tokenizer-revision` pins an explicitly selected Hugging Face tokenizer. +Hub overrides require a full commit SHA. Local tokenizer directories are +fingerprinted before and after profiling, including their file contents. +Lance and Phi4 provide `families//tests/aiperf/prepare_tokenizer.py` +to export their existing tokenizers into a standalone client directory. +Source identity is captured after acquiring the GPU lock, including content +fingerprints for tracked changes and untracked files. Generated run output is +excluded. Changes to source, tokenizer, bundle or inputs between the snapshots +preserve raw evidence and prevent a success summary. + Use the existing sentence/ranking examples or prepared audio/image JSONL from the guides below. The command verifies every corpus input against native output, then checks all 55 complete responses from five warmups and 50 measured requests, -including SHA-256 hashes of media bytes, complete vectors/transcripts/boxes, +including SHA-256 hashes of media bytes, complete vectors/transcripts/boxes/answers, native class IDs or passage indices, and every request/timing join. Results separate HTTP latency from public Task time. Build/load/startup, native/HTTP verification and warmup are excluded from measured throughput. This transport verification is @@ -448,6 +460,18 @@ separate from the owning family's numerical/quality criteria. It establishes no capacity or performance regression gate. The client is AIPerf 0.13.0 from PyPI; an AIPerf source checkout is unnecessary. +DeepSeek-OCR-2 preserves aspect ratio with mean-color padding in its existing +768-pixel, 145-visual-token single-view profile. Rebuild older square-warping +bundles with the current family runtime. Its unchanged owning reference uses +1024-pixel crop-mode inference; record that different visual budget when +interpreting conversion parity. This endpoint run measures the native Task +and does not claim a reference-relative speedup. + +Native InternVL3 retains one 448-pixel, 256-visual-token view. Its unchanged +owning reference may tile one image into multiple views. Record that visual +budget difference alongside conversion parity; serving one original image +does not establish identical internal view counts. + ## Native endpoint pilot coverage These original deliveries are historical pilot evidence. Use the stage issues @@ -494,7 +518,8 @@ through AIPerf. This qualifies object detection; document/table layout remains p Use [Profile Embeddings with AIPerf](../../website/docs/user-guides/profile-embeddings-with-aiperf.md), [Profile Reranking with AIPerf](../../website/docs/user-guides/profile-reranking-with-aiperf.md), [Profile Transcription with AIPerf](../../website/docs/user-guides/profile-transcription-with-aiperf.md), -or [Profile Object Detection with AIPerf](../../website/docs/user-guides/profile-detection-with-aiperf.md) +[Profile Object Detection with AIPerf](../../website/docs/user-guides/profile-detection-with-aiperf.md), +or [Profile Image Understanding with AIPerf](../../website/docs/user-guides/profile-vision-with-aiperf.md) for the short recipes. Native Task-call wall time stays separate from client HTTP latency; these small runs are integration examples, not performance gates. diff --git a/apps/aiperf_qual/examples/prepare_vision_input.py b/apps/aiperf_qual/examples/prepare_vision_input.py new file mode 100644 index 0000000000..ff40010d3c --- /dev/null +++ b/apps/aiperf_qual/examples/prepare_vision_input.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Write one image/question pair for AIPerf without recompressing the image.""" + +import argparse +import base64 +import io +import json +from pathlib import Path + +from PIL import Image + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image", type=Path, required=True) + parser.add_argument("--prompt", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + raw = args.image.read_bytes() + with Image.open(io.BytesIO(raw)) as image: + if image.mode != "RGB" or image.format not in {"PNG", "JPEG"} or getattr(image, "n_frames", 1) != 1: + parser.error("image must be one RGB PNG or JPEG frame") + mime = "jpeg" if image.format == "JPEG" else "png" + image.verify() + if not args.prompt.strip(): + parser.error("prompt must not be empty") + row = {"session_id": "image-question", "texts": [args.prompt], + "image": f"data:image/{mime};base64," + base64.b64encode(raw).decode("ascii")} + with args.output.open("x") as output: + output.write(json.dumps(row) + "\n") + + +if __name__ == "__main__": + main() diff --git a/apps/aiperf_qual/tests/test_endpoint_profile.py b/apps/aiperf_qual/tests/test_endpoint_profile.py index dc3ca8b3d1..4ab5708009 100644 --- a/apps/aiperf_qual/tests/test_endpoint_profile.py +++ b/apps/aiperf_qual/tests/test_endpoint_profile.py @@ -3,6 +3,7 @@ import base64 import json +from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace @@ -26,6 +27,25 @@ def test_invalid_or_nonendpoint_profile_has_a_clear_selection_error(tmp_path, mo assert not (tmp_path / "uncreated").exists() +@pytest.mark.parametrize("revision", ["main", "v1.0", ""]) +def test_revision_only_override_is_validated_before_starting_a_profile(tmp_path, monkeypatch, revision): + import trtmc_benchmark.catalog as catalog + from trtmc_aiperf_qual import endpoint_profile + + entry = SimpleNamespace(name="vision", hf_id="organization/checkpoint", + model=SimpleNamespace(task="vision_language_generation", hf_revision="a" * 40)) + monkeypatch.setattr(catalog, "ManifestCatalog", lambda _: SimpleNamespace(entries=lambda: [entry])) + monkeypatch.setattr(endpoint_profile, "inventory", lambda *_: [{"profile": "vision"}]) + monkeypatch.setattr(endpoint_profile, "gpu_exclusive", lambda _: nullcontext()) + monkeypatch.setattr(endpoint_profile, "source_identity", + lambda *_: pytest.fail("invalid tokenizer revision reached profiling")) + environment = Environment({"repo": str(tmp_path), "runtime_root": str(tmp_path), + "aiperf_python": "unused"}) + with pytest.raises(ConfigError, match="full commit SHA"): + endpoint_profile.profile(environment, "vision", Path("unused"), Path("unused"), + tmp_path / "out", tokenizer_revision=revision) + + def test_singleton_embedding_wire_forms_keep_the_same_input_identity(): assert input_key("embeddings", {"input": "hello"}) == input_key("embeddings", {"input": ["hello"]}) with pytest.raises(ValueError): @@ -87,3 +107,32 @@ def test_transcription_check_requires_the_complete_native_text(): assert check_output("transcription", {"text": "Full transcript."}, {"text": "Full transcript."}) == 1 with pytest.raises(ValueError): check_output("transcription", {"text": "Full"}, {"text": "Full transcript."}) + + +def test_vision_checks_complete_answers_token_counts_and_image_question_identity(tmp_path): + image = "data:image/png;base64," + base64.b64encode(b"original image").decode() + inputs = tmp_path / "inputs.jsonl" + inputs.write_text(json.dumps({"texts": ["Question?"], "image": image}) + "\n") + row, = samples("vision", inputs, "model", 10) + payload = row["payload"] + other = json.loads(json.dumps(payload)) + other["messages"][0]["content"][0]["text"] = "Different question?" + assert input_key("vision", payload) != input_key("vision", other) + other = json.loads(json.dumps(payload)) + other["messages"][0]["content"][1]["image_url"]["url"] = image + "AA==" + with pytest.raises(ValueError): + input_key("vision", other) + native = {"text": "Full answer.", "output_tokens": 3} + response = {"choices": [{"index": 0, "message": {"role": "assistant", "content": "Full answer."}}], + "usage": {"completion_tokens": 3}} + assert check_output("vision", response, native) == len(native["text"]) + response["choices"][0]["message"]["content"] = "Full" + with pytest.raises(ValueError): + check_output("vision", response, native) + response["choices"][0]["message"]["content"] = "" + with pytest.raises(ValueError): + check_output("vision", response, {"text": "", "output_tokens": 3}) + response["choices"][0]["message"]["content"] = "Full answer." + response["usage"]["completion_tokens"] = 2 + with pytest.raises(ValueError): + check_output("vision", response, native) diff --git a/apps/aiperf_qual/tests/test_endpoint_provenance.py b/apps/aiperf_qual/tests/test_endpoint_provenance.py new file mode 100644 index 0000000000..2755453ba2 --- /dev/null +++ b/apps/aiperf_qual/tests/test_endpoint_provenance.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import subprocess + +import pytest + +from trtmc_aiperf_qual.config import ConfigError +from trtmc_aiperf_qual.endpoint_profile import source_identity, tokenizer_identity + + +@pytest.fixture +def repository(tmp_path): + root = tmp_path / "repository" + root.mkdir() + for arguments in (["init", "-q"], ["config", "user.name", "Test"], + ["config", "user.email", "test@example.com"]): + subprocess.run(["git", *arguments], cwd=root, check=True, capture_output=True) + (root / "source.py").write_text("value = 0\n") + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "--no-gpg-sign", "-qm", "fixture"], cwd=root, check=True) + return root + + +def test_changes_to_an_already_dirty_tracked_file_change_identity(repository): + source = repository / "source.py" + source.write_text("value = 1\n") + before = source_identity(repository, repository / "output") + source.write_text("value = 2\n") + after = source_identity(repository, repository / "output") + assert before["source_head"] == after["source_head"] + assert before["source_status"] == after["source_status"] + assert before["source_contents_sha256"] != after["source_contents_sha256"] + + +def test_untracked_source_content_changes_are_detected(repository): + source = repository / "new_source.py" + source.write_text("value = 1\n") + before = source_identity(repository, repository / "output") + source.write_text("value = 2\n") + after = source_identity(repository, repository / "output") + assert before["source_status"] == after["source_status"] + assert before["source_contents_sha256"] != after["source_contents_sha256"] + + +def test_generated_output_does_not_change_source_identity(repository): + output = repository / "output [run]" + output.mkdir() + before = source_identity(repository, output) + (output / "nested").mkdir() + (output / "nested/records.json").write_text('{"requests": 55}\n') + assert source_identity(repository, output) == before + + +def test_local_tokenizer_files_are_identified_by_contents(tmp_path): + directory = tmp_path / "tokenizer" + directory.mkdir() + config = directory / "tokenizer.json" + config.write_text('{"version": 1}\n') + before = tokenizer_identity(str(directory), None, override=True) + config.write_text('{"version": 2}\n') + after = tokenizer_identity(str(directory), None, override=True) + assert before["name"] == after["name"] and before["revision"] is after["revision"] is None + assert before["files_sha256"] != after["files_sha256"] + + +@pytest.mark.parametrize("revision", [None, "main", "v1.0"]) +def test_remote_tokenizer_override_requires_an_immutable_revision(revision): + with pytest.raises(ConfigError, match="full commit SHA"): + tokenizer_identity("organization/tokenizer", revision, override=True) + pin = "a" * 40 + assert tokenizer_identity("organization/tokenizer", pin, override=True) == { + "name": "organization/tokenizer", "revision": pin, + } diff --git a/apps/aiperf_qual/trtmc_aiperf_qual/cli.py b/apps/aiperf_qual/trtmc_aiperf_qual/cli.py index 6478433e43..728aeffb9b 100644 --- a/apps/aiperf_qual/trtmc_aiperf_qual/cli.py +++ b/apps/aiperf_qual/trtmc_aiperf_qual/cli.py @@ -430,7 +430,7 @@ def main(argv: Sequence[str] | None = None) -> int: text.add_argument("--out", type=Path, required=True) endpoints = commands.add_parser("endpoint-models", help="inventory every checkpoint for a native endpoint group") endpoints.add_argument("--repo", type=Path, required=True) - endpoints.add_argument("--group", choices=("embeddings", "reranking", "transcription", "detection"), required=True) + endpoints.add_argument("--group", choices=("embeddings", "reranking", "transcription", "detection", "vision"), required=True) endpoint_run = commands.add_parser("endpoint-profile", help="verify and profile one built native endpoint model") endpoint_run.add_argument("--environment", type=Path, required=True) endpoint_run.add_argument("--profile", required=True) @@ -438,6 +438,8 @@ def main(argv: Sequence[str] | None = None) -> int: endpoint_run.add_argument("--inputs", type=Path, required=True) endpoint_run.add_argument("--out", type=Path, required=True) endpoint_run.add_argument("--max-output-tokens", type=int, default=120) + endpoint_run.add_argument("--tokenizer", help="AIPerf client tokenizer name or prepared local directory") + endpoint_run.add_argument("--tokenizer-revision", help="revision of an explicitly selected client tokenizer") arguments = parser.parse_args(argv) try: if arguments.command == "endpoint-profile": @@ -445,7 +447,8 @@ def main(argv: Sequence[str] | None = None) -> int: result = endpoint_profile(load_environment(arguments.environment), arguments.profile, arguments.bundle, arguments.inputs, arguments.out, - token_limit=arguments.max_output_tokens) + token_limit=arguments.max_output_tokens, tokenizer=arguments.tokenizer, + tokenizer_revision=arguments.tokenizer_revision) print(json.dumps(result, indent=2)) return 0 if arguments.command == "endpoint-models": diff --git a/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_catalog.py b/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_catalog.py index b633d522b2..507aaa181c 100644 --- a/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_catalog.py +++ b/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_catalog.py @@ -24,6 +24,7 @@ "transcription": {"transcription", "speech_transcription", "transcription_streaming", "streaming_speech_transcription"}, "detection": {"object_detection", "image_to_boxes"}, + "vision": {"vision_language_generation", "images_text_to_text"}, } diff --git a/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_profile.py b/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_profile.py index 94b79b1f2e..fc78c23f64 100644 --- a/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_profile.py +++ b/apps/aiperf_qual/trtmc_aiperf_qual/endpoint_profile.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Profile an existing native bundle through the standard non-generation endpoints. +"""Profile an existing native bundle through standard single-input endpoints. Reuse the persistent server, benchmark worker, published AIPerf runner, and request/metric joins. Family correctness remains a separate prerequisite; @@ -14,6 +14,9 @@ import hashlib import json import math +import os +import re +import subprocess import urllib.request import uuid from pathlib import Path @@ -31,9 +34,61 @@ "reranking": ("nim_rankings", "/v1/ranking"), "transcription": ("audio_transcription", "/v1/audio/transcriptions"), "detection": ("image_retrieval", "/v1/infer"), + "vision": ("chat", "/v1/chat/completions"), } +def source_identity(repository: Path, output: Path) -> dict[str, str]: + """Fingerprint Git contents, including an already-dirty working tree.""" + paths = ["--", "."] + try: + relative = output.resolve().relative_to(repository.resolve()) + except ValueError: + pass + else: + paths.append(f":(top,literal,exclude){relative.as_posix()}") + delta = subprocess.check_output( + ["git", "diff", "HEAD", "--binary", "--no-ext-diff", "--no-textconv", *paths], cwd=repository, + ) + untracked = subprocess.check_output( + ["git", "ls-files", "--others", "--exclude-standard", "-z", *paths], cwd=repository, + ) + files = {} + for raw in sorted(name for name in untracked.split(b"\0") if name): + name = os.fsdecode(raw) + path = repository / name + files[name] = {"link": os.readlink(path) if path.is_symlink() else None, + "sha256": digest(path) if path.is_file() else None} + contents = json.dumps({"tracked_delta": hashlib.sha256(delta).hexdigest(), "untracked": files}, + sort_keys=True).encode() + return {"source_head": git(repository, "rev-parse", "HEAD"), + "source_status": git(repository, "status", "--porcelain", "--untracked-files=all", *paths), + "source_contents_sha256": hashlib.sha256(contents).hexdigest()} + + +def tokenizer_identity(name: str, revision: str | None, *, override: bool) -> dict[str, Any]: + """Identify local tokenizer contents or an immutable Hub override.""" + identity = {"name": name, "revision": revision} + directory = Path(name).expanduser() + if directory.is_dir(): + files = {} + for path in sorted(directory.rglob("*")): + if path.is_symlink() and path.is_dir(): + raise ConfigError("local tokenizer directories cannot contain directory symlinks") + if path.is_dir(): + continue + if not path.is_file(): + raise ConfigError("local tokenizer contents must be readable regular files") + files[path.relative_to(directory).as_posix()] = { + "link": os.readlink(path) if path.is_symlink() else None, "sha256": digest(path), + } + identity.update(local_path=str(directory.resolve()), + files_sha256=hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest()) + elif override and (revision is None or re.fullmatch(r"[0-9a-fA-F]{40}", revision) is None): + raise ConfigError("a Hub tokenizer override requires --tokenizer-revision with a full commit SHA") + return identity + + def input_key(group: str, payload: dict[str, Any]) -> str: """Identify the effective wire input, including original media bytes.""" if group == "transcription": @@ -45,6 +100,13 @@ def input_key(group: str, payload: dict[str, Any]) -> str: if isinstance(text, list): text, = text value = json.dumps(text, ensure_ascii=False).encode() + elif group == "vision": + message, = payload["messages"] + parts = message["content"] + image, = [part["image_url"]["url"] for part in parts if part["type"] == "image_url"] + image_digest = hashlib.sha256(base64.b64decode(image.split(",", 1)[1], validate=True)).hexdigest() + text = "".join(part["text"] for part in parts if part["type"] == "text") + value = json.dumps([text, image_digest], ensure_ascii=False).encode() else: value = json.dumps([payload["query"], payload["passages"]], sort_keys=True, ensure_ascii=False).encode() return hashlib.sha256(value).hexdigest() @@ -71,6 +133,12 @@ def samples(group: str, inputs: Path, model: str, token_limit: int) -> list[dict payload = {"model": model, "max_output_tokens": token_limit, "file": {"b64_data": base64.b64encode(path.read_bytes()).decode()}} native = {"audio_path": str(path), "max_new_tokens": token_limit, "streaming": False} + elif group == "vision": + prompt, = row["texts"] + payload = {"model": model, "messages": [{"role": "user", "content": [ + {"type": "text", "text": prompt}, {"type": "image_url", "image_url": {"url": row["image"]}}]}], + "max_tokens": token_limit, "temperature": 1.0, "top_k": 1} + native = {"prompt": prompt, "max_new_tokens": token_limit, "temperature": 1.0, "top_k": 1} else: payload = {"input": [{"type": "image_url", "url": row["image"]}], "model": model} native = {} # The caller writes exactly these embedded bytes for the native image decoder. @@ -102,6 +170,14 @@ def post(url: str, payload: dict[str, Any], request_id: str, group: str) -> Any: def check_output(group: str, actual: Any, native: dict[str, Any], dimensions=None) -> int: """Preserve the complete public Task output, not a summary or a success code.""" + if group == "vision": + choice, = actual["choices"] + text = choice["message"]["content"] + if (not isinstance(text, str) or not text.strip() or text != native["text"] or + choice["message"]["role"] != "assistant" or choice["index"] != 0 or + actual["usage"]["completion_tokens"] != native["output_tokens"]): + raise ValueError("HTTP answer or token count differs from the complete native result") + return len(text) if group == "transcription": if actual["text"] != native["text"]: raise ValueError("HTTP transcript differs from the complete native transcript") @@ -150,7 +226,8 @@ def check_output(group: str, actual: Any, native: dict[str, Any], dimensions=Non def profile(environment: Environment, name: str, bundle: Path, inputs: Path, out: Path, - *, token_limit: int = 120) -> dict[str, Any]: + *, token_limit: int = 120, tokenizer: str | None = None, + tokenizer_revision: str | None = None) -> dict[str, Any]: """One serial model run: native verification, then 5 warmups and 50 measured requests.""" from trtmc_benchmark.catalog import ManifestCatalog from trtmc_perf_serving.backends.trtmc import TrtmcWorkerBackend @@ -165,16 +242,21 @@ def profile(environment: Environment, name: str, bundle: Path, inputs: Path, out raise ConfigError("select the group's normal full-model checkpoint profile") out.mkdir(parents=True, exist_ok=False) client = str(environment["aiperf_python"]) - client_info = check_aiperf(environment, {}) runtime = environment.path("runtime_root") - provenance = {"source_head": git(repo, "rev-parse", "HEAD"), - "source_status": git(repo, "status", "--porcelain"), "profile": name, - "manifest": json.loads(entry.model.manifest_path.read_text()), - "bundle_sha256": digest(bundle), "inputs_sha256": digest(inputs), "aiperf": client_info} - (out / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n") - corpus = samples(group, inputs, entry.hf_id, token_limit) expected = {} with gpu_exclusive(environment): + # Capture the source that actually starts inference, after any lock wait. + revision = (tokenizer_revision if tokenizer_revision is not None else + entry.model.hf_revision if tokenizer is None else None) + tokenizer_override = tokenizer is not None or tokenizer_revision is not None + selected_tokenizer = tokenizer_identity(tokenizer or entry.hf_id, revision, override=tokenizer_override) + provenance = {**source_identity(repo, out), "profile": name, + "manifest": json.loads(entry.model.manifest_path.read_text()), + "bundle_sha256": digest(bundle), "inputs_sha256": digest(inputs), + "aiperf": check_aiperf(environment, {}), + "tokenizer": selected_tokenizer} + (out / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n") + corpus = samples(group, inputs, entry.hf_id, token_limit) native = TrtmcWorkerBackend(worker=runtime / "trtmc_benchmark_worker", session={ "schema_version": 2, "operation": entry.operation, "bundle": str(bundle), "runtime_root": str(runtime), "request": {}, "expected_family": entry.family, @@ -182,15 +264,17 @@ def profile(environment: Environment, name: str, bundle: Path, inputs: Path, out try: for row in corpus: dimensions = None - if group == "detection": + if group in {"detection", "vision"}: from PIL import Image - raw = base64.b64decode(row["payload"]["input"][0]["url"].split(",", 1)[1], validate=True) + raw = base64.b64decode((row["payload"]["input"][0]["url"] if group == "detection" + else next(part["image_url"]["url"] for part in row["payload"]["messages"][0]["content"] + if part["type"] == "image_url")).split(",", 1)[1], validate=True) path = out / (row["key"] + ".image") path.write_bytes(raw) with Image.open(path) as image: dimensions = image.size - row["request"] = {"image_path": str(path)} + row["request"]["image_path"] = str(path) observed = native.invoke(row["request"], out / row["key"]).observation expected[row["key"]] = {"observation": observed, "dimensions": dimensions} finally: @@ -209,6 +293,12 @@ def profile(environment: Environment, name: str, bundle: Path, inputs: Path, out "--warmup-request-count", "5", "--warmup-concurrency", "1", "--request-count", "50"] if group == "transcription": arguments += ["--extra-inputs", f"max_output_tokens:{token_limit}"] + if group == "vision": + arguments += ["--extra-inputs", f"max_tokens:{token_limit}", "temperature:1.0", "top_k:1"] + if tokenizer: + arguments += ["--tokenizer", selected_tokenizer.get("local_path", tokenizer)] + if revision: + arguments += ["--tokenizer-revision", revision] run = run_aiperf(environment, out / "client", arguments, model_name=entry.hf_id, command_prefix=[client, "-m", "aiperf"], timeout_s=1800) if run.exit_code: @@ -246,7 +336,16 @@ def profile(environment: Environment, name: str, bundle: Path, inputs: Path, out summary.update(measured_audio_seconds=seconds, audio_seconds_per_second=seconds / elapsed) if group == "reranking": summary["passages_per_second"] = sum(r["server"]["input_items"] for r in measured) / elapsed - if group == "detection": + if group in {"detection", "vision"}: summary["images_per_second"] = summary["requests_per_second"] + final_source = source_identity(repo, out) + provenance["final_source"] = final_source + final_tokenizer = tokenizer_identity(tokenizer or entry.hf_id, revision, override=tokenizer_override) + provenance["final_tokenizer"] = final_tokenizer + (out / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n") + if (any(final_source[key] != provenance[key] for key in final_source) or + final_tokenizer != selected_tokenizer or + digest(bundle) != provenance["bundle_sha256"] or digest(inputs) != provenance["inputs_sha256"]): + raise ConfigError("source, tokenizer, bundle, or inputs changed during profiling; raw evidence is preserved") (out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") return summary diff --git a/apps/server/native_worker.cpp b/apps/server/native_worker.cpp index 4c2babcaf8..e820146b0c 100644 --- a/apps/server/native_worker.cpp +++ b/apps/server/native_worker.cpp @@ -11,6 +11,7 @@ #include "trtmc/audio.hpp" #include "trtmc/control.hpp" #include "trtmc/features.hpp" +#include "trtmc/language.hpp" #include "trtmc/perception.hpp" #include "trtmc/runtime/family_loader.h" #include "trtmc/stream.hpp" @@ -447,9 +448,64 @@ int run_detection_protocol(Invoke&& invoke, Extract&& extract, std::istream& inp input, output); } +template +int run_vision_protocol(std::int64_t default_tokens, Generate&& generate, std::istream& input, + std::ostream& output) { + return run_protocol({{"event", "ready"}, + {"protocol_version", 2}, + {"capabilities", Json::array({"vision_language_generation"})}, + {"default_max_new_tokens", default_tokens}}, + {{"generate_vision", + [&](const Json& request) { + const auto prompt = string_field(request, "prompt"); + cli::io::LoadedImage image; + const auto path = string_field(request, "image_path"); + try { + image = cli::io::read_image(path); + } catch (const std::exception&) { + throw ProtocolError("image must be a readable image file"); + } + if (image.height <= 0 || image.width <= 0 || image.pixels.empty()) + throw ProtocolError("image must contain RGB pixels"); + auto result = generate(prompt, image, request); + result["image_height"] = image.height; + result["image_width"] = image.width; + return result; + }}}, + input, output); +} + int run_existing_worker(ITask& task, std::istream& input, std::ostream& output) { // Legacy bundle modes select the declared public Task before invocation. // Several families implement more C++ interfaces than a given bundle enables. + if (std::string_view(task.task()) == IVisionLanguageGeneration::kTask) { + auto* vision = dynamic_cast(&task); + if (!vision) + throw std::invalid_argument("bundle does not implement vision-language generation"); + const auto* text = dynamic_cast(&task); + const auto defaults = text ? text->default_max_new_tokens() : 128; + return run_vision_protocol( + defaults, + [&](const std::string& prompt, const cli::io::LoadedImage& image, const Json& request) { + const auto config = parse_config(request, defaults); + if (!config.system_prompt.empty()) + throw ProtocolError("system prompts are unsupported for image requests"); + const auto started = std::chrono::steady_clock::now(); + const auto result = vision->generate(prompt, image.pixels.data(), image.height, + image.width, config); + const auto wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started) + .count(); + return Json{{"text", result.text}, + {"completion_tokens", result.token_ids.size()}, + {"setup_ms", result.setup_ms}, + {"prefill_ms", result.prefill_ms}, + {"decode_ms", result.decode_ms}, + {"model_call_ms", wall_ms}, + {"timing_scope", "public_task_call_wall"}}; + }, + input, output); + } if (std::string_view(task.task()) == IEncoding::kTask) { auto* encoder = dynamic_cast(&task); if (!encoder) @@ -528,6 +584,34 @@ int run_existing_worker(ITask& task, std::istream& input, std::ostream& output) } int run_model_worker(const Model& model, std::istream& input, std::ostream& output) { + const auto primary = model.info().bundle_task; + if (primary == ImagesTextToText::kTask || primary == IVisionLanguageGeneration::kTask) { + const auto task = model.task(); + const auto fields = task.config_fields(); + return run_vision_protocol( + default_token_limit(fields), + [&](const std::string& prompt, const cli::io::LoadedImage& image, const Json& request) { + const auto config = parse_sdk_config(request, fields); + const auto input = ImagesTextToTextRequest::from_parts( + {ImageInput{{image.pixels.data(), image.pixels.size()}, + static_cast(image.height), + static_cast(image.width)}, + TextPart{prompt}}); + const auto started = std::chrono::steady_clock::now(); + const auto result = task.run(input, config); + const auto wall_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started) + .count(); + return Json{{"text", std::string(result.text())}, + {"completion_tokens", result.token_ids().size()}, + {"setup_ms", result.setup_ms()}, + {"prefill_ms", result.prefill_ms()}, + {"decode_ms", result.decode_ms()}, + {"model_call_ms", wall_ms}, + {"timing_scope", "public_task_call_wall"}}; + }, + input, output); + } if (model.supports()) { const auto task = model.task(); return run_embedding_protocol([&](const std::string& text) { return task.run({text}); }, diff --git a/apps/server/python/tests/test_vision.py b/apps/server/python/tests/test_vision.py new file mode 100644 index 0000000000..4c41228a97 --- /dev/null +++ b/apps/server/python/tests/test_vision.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import base64 +from concurrent.futures import Future +import io +import json +from pathlib import Path + +from fastapi.testclient import TestClient +from PIL import Image +import pytest + +from trtmc_server.app import ServerConfig, create_app +from trtmc_server.errors import WorkerCrashedError, WorkerSaturatedError +from trtmc_server.registry import ModelRegistry + + +class Registry: + def __init__(self): + data = io.BytesIO() + Image.new("RGB", (8, 4), color="red").save(data, format="PNG") + self.image = data.getvalue() + self.paths = [] + self.calls = 0 + self.closed = 0 + self.retired = False + self.failure = None + self.result = {"text": "Red. Complete answer.", "completion_tokens": 3, + "image_height": 4, "image_width": 8, + "model_call_ms": 12.5, "timing_scope": "public_task_call_wall"} + + def start(self): + pass + + def close(self): + pass + + def max_tokens(self, model, cap): + return min(10, cap) + + def supports(self, model, capability): + return model == "vision" and capability == "vision_language_generation" + + def acquire(self, model): + if self.failure == "busy": + raise WorkerSaturatedError("busy") + return Session(self) + + +class Session: + replica_name = "vision-1" + + def __init__(self, registry): + self.registry = registry + + def submit(self, operation, payload): + path = Path(payload["image_path"]) + self.registry.paths.append(path) + assert path.read_bytes() == self.registry.image + assert operation == "generate_vision" and payload["prompt"] == "Color?" + assert payload["config"] == {"max_new_tokens": 10, "temperature": 1.0, "top_k": 1} + self.registry.calls += 1 + if self.registry.failure == "crashed": + raise WorkerCrashedError("private path") + result = Future() + result.set_result(self.registry.result) + return result + + def close(self): + self.registry.closed += 1 + + def retire(self, error): + self.registry.retired = True + result = Future() + result.set_result(None) + return result + + +def payload(registry): + return {"model": "vision", "max_tokens": 10, "temperature": 1.0, "top_k": 1, + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "Color?"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + + base64.b64encode(registry.image).decode()}}]}]} + + +def client_for(registry, records=None, **limits): + return TestClient(create_app(registry, ServerConfig(max_body_bytes=16000, max_prompt_bytes=128, + max_generation_tokens=128, records=records, **limits))) + + +def test_image_question_full_answer_and_request_timing(tmp_path): + registry = Registry() + records = tmp_path / "requests.jsonl" + ModelRegistry._validate_ready({"event": "ready", "protocol_version": 2, + "capabilities": ["vision_language_generation"]}) + with client_for(registry, records) as client: + response = client.post("/v1/chat/completions", json=payload(registry), headers={"X-Request-ID": "vqa-42"}) + assert response.status_code == 200 and response.headers["x-request-id"] == "vqa-42" + body = response.json() + assert body["choices"][0]["message"] == {"role": "assistant", "content": registry.result["text"]} + assert body["usage"]["completion_tokens"] == 3 and body["object"] == "chat.completion" + record = json.loads(records.read_text()) + assert record["request_id"] == "vqa-42" and record["image_width"] == 8 and record["image_height"] == 4 + assert record["image_bytes"] == len(registry.image) and record["model_call_ms"] == 12.5 + assert record["timing_scope"] == "public_task_call_wall" and record["streaming"] == "none" + assert registry.closed == 1 and all(not path.exists() for path in registry.paths) + + +@pytest.mark.parametrize("kind", ["two_images", "two_turns", "system", "no_text", "stream", "stop", "thinking", "n", + "external_url", "local_path", "bad_media", "other_model", "tokens", "duplicate_tokens"]) +def test_unsupported_image_requests_never_invoke_native(kind): + registry = Registry() + body = payload(registry) + message = body["messages"][0] + if kind == "two_images": + message["content"].append(message["content"][-1]) + elif kind in {"system", "two_turns"}: + body["messages"].insert(0, {"role": "system" if kind == "system" else "user", "content": "Hi"}) + elif kind == "no_text": + message["content"].pop(0) + elif kind in {"external_url", "local_path", "bad_media"}: + message["content"][-1]["image_url"]["url"] = { + "external_url": "https://example.org/test.png", "local_path": "/etc/passwd", + "bad_media": "data:image/png;base64,???"}[kind] + else: + field, value = {"stream": ("stream", True), "stop": ("stop", "red"), "n": ("n", 2), + "thinking": ("enable_thinking", False), "other_model": ("model", "text-only"), + "tokens": ("max_tokens", 129), "duplicate_tokens": ("max_completion_tokens", 10)}[kind] + body[field] = value + with client_for(registry) as client: + response = client.post("/v1/chat/completions", json=body) + assert response.status_code == 400 and registry.calls == 0 + assert all(not path.exists() for path in registry.paths) + + +@pytest.mark.parametrize("text", ["", " "]) +def test_empty_native_text_is_preserved_for_the_family_to_grade(text): + registry = Registry() + registry.result.update(text=text, completion_tokens=0) + with client_for(registry) as client: + response = client.post("/v1/chat/completions", json=payload(registry)) + assert response.status_code == 200 + assert response.json()["choices"][0]["message"]["content"] == text + assert response.json()["usage"]["completion_tokens"] == 0 and not registry.retired + assert registry.closed == 1 and all(not path.exists() for path in registry.paths) + + +@pytest.mark.parametrize("patch", [{"image_width": 7}, {"image_height": True}, + {"timing_scope": "kernel"}, {"completion_tokens": True}, + {"model_call_ms": -1}]) +def test_invalid_native_result_retires_worker_and_releases_upload(patch): + registry = Registry() + registry.result.update(patch) + with client_for(registry) as client: + response = client.post("/v1/chat/completions", json=payload(registry)) + assert response.status_code == 502 and registry.retired and registry.closed == 1 + assert all(not path.exists() for path in registry.paths) + + +@pytest.mark.parametrize("failure,status", [("busy", 429), ("crashed", 503), ("pixels", 413)]) +def test_image_failure_records_and_cleanup(tmp_path, failure, status): + registry = Registry() + registry.failure = failure + records = tmp_path / "requests.jsonl" + with client_for(registry, records, max_image_pixels=16 if failure == "pixels" else 32) as client: + response = client.post("/v1/chat/completions", json=payload(registry)) + assert response.status_code == status and "private" not in response.text + record = json.loads(records.read_text()) + assert record["status"] == status and record["image_bytes"] == len(registry.image) + assert all(not path.exists() for path in registry.paths) diff --git a/apps/server/python/trtmc_server/app.py b/apps/server/python/trtmc_server/app.py index 1d5907adc8..36205f5842 100644 --- a/apps/server/python/trtmc_server/app.py +++ b/apps/server/python/trtmc_server/app.py @@ -32,7 +32,7 @@ WorkerTimeoutError, ) from .metrics import Metrics -from .protocol import chat_prompt, extract_embedding, extract_ranking, extract_result, extract_transcription, extract_detection, generation_config, public_worker_error +from .protocol import chat_prompt, chat_image, extract_vision_result, extract_embedding, extract_ranking, extract_result, extract_transcription, extract_detection, generation_config, public_worker_error from .records import RequestPolicy, RequestRecords from .registry import ModelRegistry from .schemas import ChatCompletionRequest, CompletionRequest, EmbeddingRequest, GenerationRequest, RankingRequest, TranscriptionRequest, DetectionRequest @@ -677,10 +677,67 @@ def validate(value): context["transcript_bytes"] = len(text.encode("utf-8")) return PlainTextResponse(text) if request.response_format == "text" else {"text": text} + async def vision_chat(http_request: Request, request: ChatCompletionRequest, + prompt: str, image_url: str) -> Any: + route = http_request.url.path + context = http_request.state.timing + context.update(model=request.model, input_items=1) + if (request.stream or request.stream_options is not None or request.n != 1 or + request.stop is not None or request.enable_thinking is not None): + metrics.reject(route, 400) + return error_response(400, "unsupported_parameter", + "image chat supports one nonstreaming answer without stop or thinking controls") + if not prompt.strip(): + metrics.reject(route, 400) + return error_response(400, "invalid_request", "image chat requires a text prompt", param="text") + if len(prompt.encode("utf-8")) > config.max_prompt_bytes: + metrics.reject(route, 413) + return error_response(413, "content_too_large", "prompt exceeds the server limit") + try: + default_tokens = registry.max_tokens(request.model, config.max_generation_tokens) + except ModelNotFoundError as error: + metrics.reject(route, 404) + return error_response(404, error.code, str(error), param="model") + max_tokens = request.max_completion_tokens or request.max_tokens or default_tokens + if max_tokens > config.max_generation_tokens: + metrics.reject(route, 400) + return error_response(400, "max_tokens_exceeded", "requested generation exceeds the server limit") + try: + data, suffix, metadata = await asyncio.to_thread(image_upload, image_url, config.max_image_pixels) + context.update(metadata) + except ImagePixelLimitError as error: + context.update(error.metadata) + metrics.reject(route, 413) + return error_response(413, "content_too_large", str(error)) + except ValueError as error: + metrics.reject(route, 400) + return error_response(400, "invalid_request", str(error)) + + worker_config = generation_config(request, max_tokens) + context["generation_config"] = worker_config + media = await create_uploaded_input(data, suffix) + + def validate(value): + return extract_vision_result(value, context["image_height"], context["image_width"]) + + result = await execute_task(http_request, request.model, "vision_language_generation", "generate_vision", + {"prompt": prompt, "image_path": str(media.path), "config": worker_config}, + validate, release_input=media.close) + if isinstance(result, JSONResponse): + return result + text, tokens, timings = validate(result) + context.update(timings, completion_tokens=tokens, completion_token_source="native_task") + return {"id": f"chatcmpl-{uuid.uuid4().hex}", "object": "chat.completion", + "created": int(time.time()), "model": request.model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, + "logprobs": None, "finish_reason": None}], + "usage": {"prompt_tokens": 0, "completion_tokens": tokens, "total_tokens": tokens}} + @app.post("/v1/chat/completions") async def chat_completions(http_request: Request, request: ChatCompletionRequest) -> Any: try: prompt, system_prompt = chat_prompt(request) + image_url = chat_image(request) except ValueError as error: metrics.reject("/v1/chat/completions", 400) return error_response(400, "invalid_request", str(error), param="messages") @@ -691,6 +748,8 @@ async def chat_completions(http_request: Request, request: ChatCompletionRequest "invalid_request", "max_tokens and max_completion_tokens are mutually exclusive", ) + if image_url is not None: + return await vision_chat(http_request, request, prompt, image_url) return await execute( "/v1/chat/completions", request, diff --git a/apps/server/python/trtmc_server/cli.py b/apps/server/python/trtmc_server/cli.py index 11293e5c1c..a7326bee5c 100644 --- a/apps/server/python/trtmc_server/cli.py +++ b/apps/server/python/trtmc_server/cli.py @@ -40,7 +40,7 @@ def positive_seconds(value: str) -> float: def parser() -> argparse.ArgumentParser: result = argparse.ArgumentParser( prog="trtmc-server", - description="Serve text-generation bundles alongside embedding, reranking, audio-transcription, and object-detection bundles through persistent native workers", + description="Serve text-generation bundles alongside vision-language, embedding, reranking, audio-transcription, and object-detection bundles through persistent native workers", ) result.add_argument("bundle", nargs="?", help="single .bundle artifact") result.add_argument("--model-name", help="API model name for the positional bundle") diff --git a/apps/server/python/trtmc_server/protocol.py b/apps/server/python/trtmc_server/protocol.py index 5a15243337..852f8f270b 100644 --- a/apps/server/python/trtmc_server/protocol.py +++ b/apps/server/python/trtmc_server/protocol.py @@ -10,7 +10,7 @@ from typing import Any from .errors import WorkerProtocolError, WorkerRemoteError, WorkerRequestTooLargeError -from .schemas import ChatCompletionRequest, GenerationRequest, TextContentPart +from .schemas import ChatCompletionRequest, GenerationRequest, TextContentPart, ImageContentPart def generation_config(request: GenerationRequest, max_tokens: int) -> dict[str, Any]: @@ -35,10 +35,30 @@ def chat_prompt(request: ChatCompletionRequest) -> tuple[str, str]: raise ValueError("messages must be one user message with an optional preceding system message") -def _message_text(content: str | list[TextContentPart]) -> str: +def chat_image(request: ChatCompletionRequest) -> str | None: + images = [part.image_url.url for message in request.messages if isinstance(message.content, list) + for part in message.content if isinstance(part, ImageContentPart)] + if not images: + return None + if len(images) != 1 or len(request.messages) != 1 or request.messages[0].role != "user": + raise ValueError("image chat requires one user message with exactly one image") + return images[0] + + +def _message_text(content: str | list[TextContentPart | ImageContentPart]) -> str: if isinstance(content, str): return content - return "".join(part.text for part in content) + return "".join(part.text for part in content if isinstance(part, TextContentPart)) + + +def extract_vision_result(result: Any, height: int, width: int) -> tuple[str, int, dict[str, float]]: + text, tokens, timings = extract_result(result) + if (type(result.get("image_height")) is not int or + type(result.get("image_width")) is not int or result["image_height"] != height or + result["image_width"] != width): + raise WorkerProtocolError("worker vision result has invalid text or image identity") + extract_task_timing(result) + return text, tokens, timings def extract_result(result: Any) -> tuple[str, int, dict[str, float]]: diff --git a/apps/server/python/trtmc_server/registry.py b/apps/server/python/trtmc_server/registry.py index 01aa388484..fa3f8c5ae0 100644 --- a/apps/server/python/trtmc_server/registry.py +++ b/apps/server/python/trtmc_server/registry.py @@ -179,5 +179,5 @@ def _validate_ready(metadata: dict[str, Any]) -> None: capabilities = metadata.get("capabilities") if (not isinstance(capabilities, list) or not capabilities or not all(isinstance(value, str) for value in capabilities) or - not {"text_generation", "embeddings", "reranking", "audio_transcription", "object_detection"}.intersection(capabilities)): + not {"text_generation", "embeddings", "reranking", "audio_transcription", "object_detection", "vision_language_generation"}.intersection(capabilities)): raise WorkerProtocolError("worker does not advertise a supported capability") diff --git a/apps/server/python/trtmc_server/schemas.py b/apps/server/python/trtmc_server/schemas.py index f362176128..ae49ca536c 100644 --- a/apps/server/python/trtmc_server/schemas.py +++ b/apps/server/python/trtmc_server/schemas.py @@ -19,9 +19,18 @@ class TextContentPart(StrictRequest): text: str +class ImageURL(StrictRequest): + url: str = Field(min_length=1) + + +class ImageContentPart(StrictRequest): + type: Literal["image_url"] + image_url: ImageURL + + class ChatMessage(StrictRequest): role: str - content: str | list[TextContentPart] + content: str | list[TextContentPart | ImageContentPart] class StreamOptions(StrictRequest): diff --git a/apps/server/tests/test_sdk_worker.cpp b/apps/server/tests/test_sdk_worker.cpp index 27e18bd105..ec61efc330 100644 --- a/apps/server/tests/test_sdk_worker.cpp +++ b/apps/server/tests/test_sdk_worker.cpp @@ -164,6 +164,19 @@ void test_task_selection_and_failures(const std::filesystem::path& root, run.messages[1].at("result").at("text") == item.second, "primary Task selection does not inject undeclared defaults or languages"); } + for (const auto& item : std::vector>{ + {"text_conversation", "language_fixture"}, {"images_text_to_text", "text_fixture"}}) { + std::istringstream input; + std::ostringstream output; + try { + trtmc::server::run_bundle_worker(write_bundle(root, item.first, item.second).string(), + options, input, output); + throw std::runtime_error("server selected a secondary Task instead of the primary"); + } catch (const trtmc::Error& error) { + require(error.code() == TRTMC_UNSUPPORTED && output.str().empty(), + "secondary capabilities cannot enable or replace a bundle's primary Task"); + } + } const auto failure = invoke(write_bundle(root, "must_not_run"), options, {generate("failure"), generate("must-not-retry")}); require(failure.status == 1 && failure.messages.size() == 2 && @@ -274,6 +287,36 @@ void test_detection_task(const std::filesystem::path& root, const trtmc::LoadOpt result.at("timing_scope") == "public_task_call_wall", "SDK defaults and complete native image detection results survive transport"); } + +void test_vision_task(const std::filesystem::path& root, const trtmc::LoadOptions& options, + const std::string& primary) { + const auto bundle = write_bundle(root, primary, "language_fixture"); + const auto path = root / "server-sdk-vision.png"; + std::vector pixels(8 * 4 * 3, 0.0F); + pixels[0] = 1.0F; + trtmc::cli::io::save_png(path.string(), pixels, 8, 4); + const Json valid{{"id", "vision"}, + {"op", "generate_vision"}, + {"image_path", path.string()}, + {"prompt", "Question?"}, + {"config", {{"suffix", "tail"}}}}; + auto invalid = valid; + invalid["config"]["unknown"] = 1; + const auto run = invoke(bundle, options, {invalid, valid}); + std::filesystem::remove(path); + require(run.status == 0 && run.messages.size() == 3 && + run.messages[0].at("capabilities") == Json::array({"vision_language_generation"}) && + run.messages[1].at("error").at("type") == "invalid_request_error", + "SDK vision capability and family-declared Config are honored"); + const auto& result = run.messages[2].at("result"); + const auto text = result.at("text").get(); + require(text.find("I:1;") != std::string::npos && + text.find("T:Question?;") != std::string::npos && text.size() >= 4 && + text.compare(text.size() - 4, 4, "tail") == 0 && result.at("image_height") == 4 && + result.at("image_width") == 8 && result.at("model_call_ms").get() >= 0 && + result.at("timing_scope") == "public_task_call_wall", + "actual public SDK Task receives image and question with its own Config"); +} } // namespace int main(int argc, char** argv) { @@ -290,6 +333,8 @@ int main(int argc, char** argv) { test_audio_task(root, options); test_audio_task_new_token_field(root, options); test_detection_task(root, options); + test_vision_task(root, options, "images_text_to_text"); + test_vision_task(root, options, "vision_language_generation"); return 0; } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/apps/server/tests/test_vision_worker.cpp b/apps/server/tests/test_vision_worker.cpp new file mode 100644 index 0000000000..4e4867ea9c --- /dev/null +++ b/apps/server/tests/test_vision_worker.cpp @@ -0,0 +1,106 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "cli/io.h" +#include "server/native_worker.h" +#include "trtmc/task.h" + +#include +#include +#include +#include +#include +#include + +namespace { +using Json = nlohmann::json; +void require(bool condition, const char* message) { + if (!condition) + throw std::runtime_error(message); +} +class Vision final : public trtmc::IVisionLanguageGeneration, public trtmc::ITextGeneration { + public: + int calls{0}; + const char* task() const noexcept override { return IVisionLanguageGeneration::kTask; } + std::int32_t default_max_new_tokens() const override { return 10; } + trtmc::TextResult generate(const std::string&, const trtmc::TextGenerationConfig&) override { + throw std::runtime_error("image request must never take the text-only interface"); + } + trtmc::TextResult generate(const std::string& prompt, const float* pixels, std::int32_t height, + std::int32_t width, + const trtmc::TextGenerationConfig& config) override { + require(prompt == "Color?" && height == 4 && width == 8 && pixels[0] == 1.0F && + pixels[1] == 0.0F, + "original RGB geometry and question reach the Task"); + require(config.max_new_tokens == 10 && config.top_k == 1 && config.seed == 42, + "generation controls reach the owning family"); + ++calls; + trtmc::TextResult result; + result.text = "Red. Complete answer."; + result.token_ids = {1, 2, 3}; + return result; + } +}; +void verify(const std::string& path) { + Vision task; + const Json valid{{"id", "vqa"}, + {"op", "generate_vision"}, + {"prompt", "Color?"}, + {"image_path", path}, + {"config", {{"max_new_tokens", 10}, {"top_k", 1}, {"seed", 42}}}}; + std::stringstream input; + auto missing = valid; + missing.erase("image_path"); + input << missing.dump() << '\n'; + auto unreadable = valid; + unreadable["id"] = "unreadable-image"; + unreadable["image_path"] = path + "/missing.png"; + input << unreadable.dump() << '\n'; + auto bad_config = valid; + bad_config["config"]["unknown"] = true; + input << bad_config.dump() << '\n'; + input << Json{{"id", "text"}, {"op", "generate"}, {"prompt", "Color?"}}.dump() << '\n'; + input << valid.dump() << '\n'; + auto default_config = valid; + default_config["id"] = "default-tokens"; + default_config["config"].erase("max_new_tokens"); + input << default_config.dump() << '\n'; + std::ostringstream output; + require(trtmc::server::run_task_worker(task, input, output) == 0, "worker completes"); + std::vector replies; + std::istringstream stream(output.str()); + for (std::string line; std::getline(stream, line);) + replies.push_back(Json::parse(line)); + require(replies.size() == 7 && task.calls == 2 && + replies[0].at("capabilities") == Json::array({"vision_language_generation"}) && + replies[0].at("default_max_new_tokens") == 10, + "declared bundle capability prevents inherited text dispatch"); + for (std::size_t i = 1; i < 5; ++i) + require(replies[i].at("error").at("type") == "invalid_request_error", + "invalid requests remain recoverable and never invoke the Task"); + const auto result = replies[5].at("result"); + require(result.at("text") == "Red. Complete answer." && result.at("completion_tokens") == 3 && + result.at("image_height") == 4 && result.at("image_width") == 8 && + result.at("model_call_ms").get() >= 0 && + result.at("timing_scope") == "public_task_call_wall", + "complete result and timing survive"); +} +} // namespace +int main() { + const auto path = std::filesystem::temp_directory_path() / + ("trtmc-vision-" + std::to_string(std::random_device{}()) + ".png"); + try { + std::vector pixels(8 * 4 * 3, 0.0F); + for (std::size_t i = 0; i < pixels.size(); i += 3) + pixels[i] = 1.0F; + trtmc::cli::io::save_png(path.string(), pixels, 8, 4); + verify(path.string()); + std::filesystem::remove(path); + } catch (const std::exception& error) { + std::filesystem::remove(path); + std::cerr << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/core/runtime/tensorrt/trt_module_impl.cpp b/core/runtime/tensorrt/trt_module_impl.cpp index bdd3a7d8d5..1b980ce425 100644 --- a/core/runtime/tensorrt/trt_module_impl.cpp +++ b/core/runtime/tensorrt/trt_module_impl.cpp @@ -257,7 +257,8 @@ void TrtModuleImpl::update_dynamic_shape(const std::string& name, BufferEntry& e dims.nbDims = static_cast(new_shape.size()); for (int32_t d = 0; d < dims.nbDims; ++d) dims.d[d] = new_shape[d]; - ctx_->setInputShape(name.c_str(), dims); + if (!ctx_->setInputShape(name.c_str(), dims)) + throw std::invalid_argument("TensorRT rejected dynamic shape for tensor '" + name + "'"); entry.shape = new_shape; } diff --git a/core/runtime/tests/test_trt_module_dynamic_input.cpp b/core/runtime/tests/test_trt_module_dynamic_input.cpp index ef28f4144f..ca9d41eb40 100644 --- a/core/runtime/tests/test_trt_module_dynamic_input.cpp +++ b/core/runtime/tests/test_trt_module_dynamic_input.cpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace { @@ -103,6 +104,33 @@ void test_backend_managed_graph_does_not_enable_nested_capture(nvinfer1::ICudaEn check(!module.cuda_graph_active(), "backend-managed graph leaves stream capture disabled"); } +void test_rejected_shape_fails_without_poisoning_the_next_request(nvinfer1::ICudaEngine& engine, + cudaStream_t stream) { + for (bool graph : {false, true}) { + trtmc::TrtModuleImpl module(&engine, engine.createExecutionContext(), stream); + if (graph) + module.enable_cuda_graph(); + float initial[] = {1.0F, 2.0F}; + (void)module.forward({{"input", trtmc::Tensor{initial, {2}, trtmc::DType::kFloat32}}}); + float oversized[] = {1.0F, 2.0F, 3.0F, 4.0F, 5.0F}; + bool rejected = false; + try { + (void)module.forward( + {{"input", trtmc::Tensor{oversized, {5}, trtmc::DType::kFloat32}}}); + } catch (const std::invalid_argument&) { + rejected = true; + } + check(rejected, "out-of-profile shapes must fail instead of returning stale-shape outputs"); + float next[] = {7.0F, 8.0F}; + const auto output = + module.forward({{"input", trtmc::Tensor{next, {2}, trtmc::DType::kFloat32}}}); + const auto& tensor = output.at("output"); + const auto* values = static_cast(tensor.data); + check(tensor.numel() == 2 && values[0] == 7.0F && values[1] == 8.0F, + "a rejected shape leaves the next valid request usable"); + } +} + void test_short_static_upload_does_not_keep_the_previous_input(nvinfer1::ICudaEngine& engine, cudaStream_t stream) { for (bool graph : {false, true}) { @@ -162,6 +190,7 @@ int main() { test_first_forward_allocates_unbound_dynamic_input(*engine, stream); test_external_dynamic_input_stays_external(*engine, stream); test_backend_managed_graph_does_not_enable_nested_capture(*engine, stream); + test_rejected_shape_fails_without_poisoning_the_next_request(*engine, stream); test_short_static_upload_does_not_keep_the_previous_input(*static_engine, stream); test_short_external_static_upload_preserves_its_tail(*static_engine, stream); cudaStreamDestroy(stream); diff --git a/families/deepseek_ocr/model.py b/families/deepseek_ocr/model.py index c5eed208af..87a67a271e 100644 --- a/families/deepseek_ocr/model.py +++ b/families/deepseek_ocr/model.py @@ -679,7 +679,7 @@ def get_vl_config(self, config: ModelConfig) -> dict | None: "num_image_pad_tokens": 145, # 144 projected + 1 view_separator "prefill_max_length": MAX_SEQUENCE_PREFILL_LENGTH, "vision_output_dim": hidden, - "preprocessor_type": "simple_chw", + "preprocessor_type": "pad_center_chw", "image_mean": [0.5, 0.5, 0.5], "image_std": [0.5, 0.5, 0.5], "interpolation": "bicubic", diff --git a/families/deepseek_ocr/runtime/CMakeLists.txt b/families/deepseek_ocr/runtime/CMakeLists.txt index d496d77d03..063d09ee2f 100644 --- a/families/deepseek_ocr/runtime/CMakeLists.txt +++ b/families/deepseek_ocr/runtime/CMakeLists.txt @@ -49,6 +49,21 @@ install(TARGETS trtmc_model_deepseek_ocr ) if(TRTMC_BUILD_TESTS) + add_executable(test_deepseek_ocr_image_padding + ../tests/cpp/test_image_padding.cpp + ) + target_include_directories(test_deepseek_ocr_image_padding PRIVATE + ${PROJECT_SOURCE_DIR} + ) + target_link_libraries(test_deepseek_ocr_image_padding PRIVATE + trtmc_model_deepseek_ocr + ) + target_compile_options(test_deepseek_ocr_image_padding PRIVATE + -Wall -Wextra -Wpedantic + ) + add_test(NAME deepseek_ocr_image_padding COMMAND test_deepseek_ocr_image_padding) + set_tests_properties(deepseek_ocr_image_padding PROPERTIES LABELS cpu) + add_executable(test_deepseek_ocr_runtime_config ../tests/cpp/test_runtime_config.cpp ) diff --git a/families/deepseek_ocr/runtime/image_preprocessor.cpp b/families/deepseek_ocr/runtime/image_preprocessor.cpp index 8ba7fee487..6f0b79fe55 100644 --- a/families/deepseek_ocr/runtime/image_preprocessor.cpp +++ b/families/deepseek_ocr/runtime/image_preprocessor.cpp @@ -11,6 +11,7 @@ #include "stb_image_resize2.h" #include +#include #include #include #include @@ -43,6 +44,13 @@ struct LoadedImage { bool ok{false}; }; +// PIL uses Python's ties-to-even rounding for contain and center placement. +static int round_even(double value) { + const auto lower = static_cast(std::floor(value)); + const auto fraction = value - lower; + return lower + (fraction > 0.5 || (fraction == 0.5 && lower % 2 != 0)); +} + // Resize raw uint8 RGB buffer to target_size x target_size using the given filter. static std::vector resize_raw(const unsigned char* raw, int width, int height, int target_size, stbir_filter filter) { @@ -220,13 +228,14 @@ load_pad_center_resize_normalize(const runtime::adapters::io::DecodedImage& imag const int target_size = config.fixed_image_size; const stbir_filter filter = resolve_stbir_filter(config.interpolation); - // Compute scaled dimensions that fit inside target_size x target_size - // This matches PIL ImageOps.pad behavior: scale to fit, then center. - const float scale_w = static_cast(target_size) / static_cast(image.width); - const float scale_h = static_cast(target_size) / static_cast(image.height); - const float scale = std::min(scale_w, scale_h); - const int new_w = std::max(1, static_cast(image.width * scale)); - const int new_h = std::max(1, static_cast(image.height * scale)); + int new_w = target_size; + int new_h = target_size; + if (image.width > image.height) + new_h = + std::max(1, round_even(static_cast(image.height) / image.width * target_size)); + else if (image.height > image.width) + new_w = + std::max(1, round_even(static_cast(image.width) / image.height * target_size)); // Resize preserving aspect ratio std::vector resized_small(static_cast(new_w) * new_h * 3); @@ -253,8 +262,8 @@ load_pad_center_resize_normalize(const runtime::adapters::io::DecodedImage& imag } // Center the resized image in the padded canvas - const int x_off = (target_size - new_w) / 2; - const int y_off = (target_size - new_h) / 2; + const int x_off = round_even((target_size - new_w) * 0.5); + const int y_off = round_even((target_size - new_h) * 0.5); for (int y = 0; y < new_h; ++y) { const unsigned char* src_row = resized_small.data() + static_cast(y) * new_w * 3; diff --git a/families/deepseek_ocr/tests/cpp/test_image_padding.cpp b/families/deepseek_ocr/tests/cpp/test_image_padding.cpp new file mode 100644 index 0000000000..ea7654cc35 --- /dev/null +++ b/families/deepseek_ocr/tests/cpp/test_image_padding.cpp @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#include "families/deepseek_ocr/runtime/image_preprocessor.h" + +#include +#include +#include + +namespace { +void verify(int width, int height, int left, int top, int right, int bottom) { + trtmc::runtime::adapters::io::DecodedImage image; + image.width = width; + image.height = height; + image.channels = 3; + image.pixels.assign(width * height * 3, 255); + trtmc::DeepseekOcrPreprocessConfig config; + config.fixed_image_size = 16; + config.preprocessor_type = "pad_center_chw"; + for (int channel = 0; channel < 3; ++channel) { + config.image_mean[channel] = 0.5F; + config.image_std[channel] = 0.5F; + } + const auto result = trtmc::deepseek_ocr_preprocess_decoded_image(image, config); + if (!result.ok || result.pixel_values.size() != 3 * 16 * 16) + throw std::runtime_error("padding must produce the declared CHW image"); + for (int channel = 0; channel < 3; ++channel) { + for (int y = 0; y < 16; ++y) { + for (int x = 0; x < 16; ++x) { + const bool content = x >= left && x < right && y >= top && y < bottom; + const float expected = content ? 1.0F : -1.0F / 255; + const auto value = result.pixel_values[channel * 256 + y * 16 + x]; + if (std::abs(value - expected) > 1e-6F) + throw std::runtime_error("geometry or mean-color padding differs from PIL"); + } + } + } +} +} // namespace + +int main() { + try { + // Independent PIL ImageOps.pad white-image bounding boxes at size16. + verify(3, 7, 4, 0, 11, 16); + verify(7, 3, 0, 4, 16, 11); + verify(13, 16, 2, 0, 15, 16); + verify(16, 13, 0, 2, 16, 15); + verify(16, 16, 0, 0, 16, 16); + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/families/deepseek_ocr/tests/test_builder_contract.py b/families/deepseek_ocr/tests/test_builder_contract.py index 144178741f..aa21997800 100644 --- a/families/deepseek_ocr/tests/test_builder_contract.py +++ b/families/deepseek_ocr/tests/test_builder_contract.py @@ -325,14 +325,14 @@ def test_deepseek_ocr_resizes_sam_position_embedding_for_768_view() -> None: np.testing.assert_allclose(resized, 1.0, atol=1e-6) -def test_deepseek_ocr_uses_official_768_single_view_contract() -> None: +def test_deepseek_ocr_uses_bounded_768_aspect_preserving_view() -> None: vl_config = model._DeepseekOcrModel().get_vl_config(_config()) assert vl_config is not None assert vl_config["fixed_image_size"] == 768 assert vl_config["num_image_pad_tokens"] == 145 assert vl_config["prefill_max_length"] == 256 - assert vl_config["preprocessor_type"] == "simple_chw" + assert vl_config["preprocessor_type"] == "pad_center_chw" @pytest.mark.parametrize( diff --git a/families/deepseek_ocr/tests/test_vision_oracle.py b/families/deepseek_ocr/tests/test_vision_oracle.py index 342f424132..67690ace2b 100644 --- a/families/deepseek_ocr/tests/test_vision_oracle.py +++ b/families/deepseek_ocr/tests/test_vision_oracle.py @@ -56,6 +56,7 @@ def test_native_pixels_follow_the_family_runtime_contract() -> None: image, { "fixed_image_size": 4, + "preprocessor_type": "simple_chw", "image_mean": [0.5, 0.5, 0.5], "image_std": [0.5, 0.5, 0.5], }, @@ -71,6 +72,17 @@ def test_vision_gate_is_the_active_half_cosine_contract() -> None: assert_vision_parity([[1.0, 0.0]], [[-1.0, 0.0]]) +def test_padded_pixels_keep_white_text_area_and_mean_color_border() -> None: + pixels = _native_pixels(Image.new("RGB", (13, 16), "white"), { + "fixed_image_size": 16, "preprocessor_type": "pad_center_chw", + "image_mean": [0.5] * 3, "image_std": [0.5] * 3, + }) + assert pixels.shape == (1, 3, 16, 16) + np.testing.assert_allclose(pixels[:, :, :, 2:15], 1.0) + np.testing.assert_allclose(pixels[:, :, :, :2], -1 / 255, atol=1e-7) + np.testing.assert_allclose(pixels[:, :, :, 15:], -1 / 255, atol=1e-7) + + @pytest.mark.parametrize("features", [[0.0, 0.0], [float("inf"), 1.0]]) def test_vision_gate_rejects_invalid_native_features(features) -> None: with pytest.raises(AssertionError): diff --git a/families/deepseek_ocr/tests/vision_oracle.py b/families/deepseek_ocr/tests/vision_oracle.py index de4db4d898..ecb5655307 100644 --- a/families/deepseek_ocr/tests/vision_oracle.py +++ b/families/deepseek_ocr/tests/vision_oracle.py @@ -28,10 +28,16 @@ def _bundle_section(bundle: Path, name: str) -> bytes: def _native_pixels(image, config: dict) -> np.ndarray: - from PIL import Image + from PIL import Image, ImageOps size = int(config["fixed_image_size"]) - resized = image.convert("RGB").resize((size, size), Image.Resampling.BICUBIC) + image = image.convert("RGB") + if config["preprocessor_type"] == "pad_center_chw": + color = tuple(int(value * 255) for value in config["image_mean"]) + resized = ImageOps.pad(image, (size, size), color=color) + else: + assert config["preprocessor_type"] == "simple_chw" + resized = image.resize((size, size), Image.Resampling.BICUBIC) pixels = np.asarray(resized, dtype=np.float32) / 255.0 mean = np.asarray(config["image_mean"], dtype=np.float32) std = np.asarray(config["image_std"], dtype=np.float32) diff --git a/families/internvl/model.py b/families/internvl/model.py index d8c899bab9..a2f1b2f628 100644 --- a/families/internvl/model.py +++ b/families/internvl/model.py @@ -463,6 +463,7 @@ def _build_native(request: "BuildRequest", writer: "BundleWriter") -> None: parallel.validate() config.raw["_model_dir"] = str(model_dir) model = _InternVLModel() + model_vl = model.get_vl_config(config) or {} weights = model.load_weights(str(model_dir), config) writer.set_header(family="internvl", task=request.task, backend=request.backend) if parallel.enabled: @@ -490,25 +491,36 @@ def _build_native(request: "BuildRequest", writer: "BundleWriter") -> None: verbose=request.verbose, parallel_config=parallel, ) - config.raw["_decoder_engine_role"] = "decode" - decode = model.build_engine( - config, - weights, - max_length, - precision=precision, - quant_ctx=None, - verbose=request.verbose, - parallel_config=parallel, - ) - config.raw.pop("_decoder_engine_role", None) - writer.add_bytes("engine.plan", decode) + # Stage serialized plans on disk before building the next one. writer.add_bytes("prefill.plan", prefill) + if int(model_vl.get("max_image_tiles", 1)) > 1: + # The dynamic-Sq graph accepts both multi-token prefill and + # single-token decode. Use that same mathematical network for + # tiled inputs; each runtime module owns its execution state. + writer.add_bytes("engine.plan", prefill) + del prefill + if int(model_vl.get("max_image_tiles", 1)) == 1: + config.raw["_decoder_engine_role"] = "decode" + decode = model.build_engine( + config, + weights, + max_length, + precision=precision, + quant_ctx=None, + verbose=request.verbose, + parallel_config=parallel, + ) + writer.add_bytes("engine.plan", decode) + del decode + config.raw.pop("_decoder_engine_role", None) vision = model.build_vision_engine( str(model_dir), config, weights, precision=precision, verbose=request.verbose ) if vision is None: raise RuntimeError("InternVL vision build returned no engine") - vl = model.get_vl_config(config) or {} + writer.add_bytes("vision.plan", vision) + del vision + vl = model_vl runtime = { "tensor_parallel_size": parallel.tp_size, "num_layers": config.num_hidden_layers, @@ -532,7 +544,6 @@ def _build_native(request: "BuildRequest", writer: "BundleWriter") -> None: # token cache; the tensor-parallel decoder keeps one tile until its # single-query attention avoids masked IAttention over >512 rows. runtime.update({"preprocessor_type": "simple_chw", "max_image_tiles": 1}) - writer.add_bytes("vision.plan", vision) runtime.update(_tokenizer_runtime_contract(model_dir)) writer.add_json("runtime.json", runtime) for filename in ( diff --git a/families/internvl/tests/cpp/test_image_normalization.cpp b/families/internvl/tests/cpp/test_image_normalization.cpp index adfb1f8477..24f25f8c99 100644 --- a/families/internvl/tests/cpp/test_image_normalization.cpp +++ b/families/internvl/tests/cpp/test_image_normalization.cpp @@ -82,12 +82,54 @@ void check_uniform_image(const trtmc::InternVlPreprocessConfig& config, std::uin check(seen[0] && seen[1] && seen[2], label + " pixels cover every channel"); } +void check_dynamic_tiles(bool portrait) { + auto document = nlohmann::json::parse(runtime_json()); + document["preprocessor_type"] = "dynamic_tiles_chw"; + document["max_image_tiles"] = 2; + // Match get_vl_config, which declares the thumbnail explicitly. + document["use_thumbnail"] = true; + document["fixed_image_size"] = 4; + document["image_mean"] = std::array{0, 0, 0}; + document["image_std"] = std::array{1, 1, 1}; + const auto config = trtmc::internvl_parse_preprocess_config(document.dump()); + trtmc::runtime::adapters::io::DecodedImage image; + image.width = portrait ? 4 : 8; + image.height = portrait ? 8 : 4; + image.channels = 3; + image.pixels.assign(image.width * image.height * 3, 0); + for (int y = 0; y < image.height; ++y) + for (int x = 0; x < image.width; ++x) { + const bool first = portrait ? y < 4 : x < 4; + image.pixels[(y * image.width + x) * 3 + (first ? 0 : 2)] = 255; + } + const auto result = trtmc::internvl_preprocess_decoded_image(image, config); + check(result.ok && result.num_tiles == 3 && result.pixel_values.size() == 144, + "two local tiles plus one thumbnail"); + if (result.pixel_values.size() != 144) + return; + for (int pixel = 0; pixel < 16; ++pixel) { + check(result.pixel_values[pixel] == 1 && result.pixel_values[32 + pixel] == 0, + "first tile is red in row-major order"); + check(result.pixel_values[48 + pixel] == 0 && result.pixel_values[80 + pixel] == 1, + "second tile is blue in row-major order"); + } + check(result.pixel_values[96] > result.pixel_values[111], + "thumbnail follows local tiles and preserves the complete image"); + auto prompt_config = config; + prompt_config.num_image_pad_tokens *= result.num_tiles; + const auto prompt = trtmc::internvl_format_prompt("Question?", prompt_config); + check(prompt.size() == 3 * 256 * config.image_token_str.size() + 9, + "prompt reserves every tile's image tokens"); +} + } // namespace int main() { const auto config = trtmc::internvl_parse_preprocess_config(runtime_json()); check_uniform_image(config, 0); check_uniform_image(config, 255); + check_dynamic_tiles(false); + check_dynamic_tiles(true); if (g_failures != 0) { std::cerr << g_failures << " InternVL image normalization check(s) failed\n"; return 1; diff --git a/families/internvl/tests/test_plan_lifetime.py b/families/internvl/tests/test_plan_lifetime.py new file mode 100644 index 0000000000..3d8398b027 --- /dev/null +++ b/families/internvl/tests/test_plan_lifetime.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep serialized engines out of RAM while building the next network.""" + +import gc +import json +from types import SimpleNamespace + +import pytest + + +@pytest.mark.parametrize("tiles", [1, 12]) +def test_each_plan_is_staged_and_released_before_the_next_build(tmp_path, monkeypatch, tiles): + pytest.importorskip("tensorrt") + from families.internvl import model + from families.internvl.edge_llm import dispatch + from tensorrt_model_connect.bundle_writer import BundleWriter + + live = set() + built = [] + + class Plan(bytes): + def __new__(cls, role): + value = super().__new__(cls, role.encode()) + value.role = role + live.add(role) + return value + + def __del__(self): + live.remove(self.role) + + def next_plan(role): + gc.collect() + assert not live, f"previous serialized engine retained during {role}: {live}" + built.append(role) + return Plan(role) + + class Model: + @staticmethod + def load_weights(*_args): + return {} + + @staticmethod + def build_engine(config, *_args, **_kwargs): + return next_plan(config.raw["_decoder_engine_role"]) + + @staticmethod + def build_vision_engine(*_args, **_kwargs): + return next_plan("vision") + + @staticmethod + def get_vl_config(_config): + return {"image_token_id": 3, "vision_output_dim": 4, "num_image_pad_tokens": 256, + "max_image_tiles": tiles} + + monkeypatch.setattr(model, "_InternVLModel", Model) + monkeypatch.setattr(model, "_tokenizer_runtime_contract", lambda _directory: {}) + monkeypatch.setattr(dispatch, "build", lambda request, writer, native: native(request, writer)) + (tmp_path / "config.json").write_text(json.dumps({ + "model_type": "internvl", "hidden_size": 4, "num_hidden_layers": 1, + "vocab_size": 8, "num_attention_heads": 1, "num_key_value_heads": 1, + })) + request = SimpleNamespace( + mtp_seq_len=None, dynamic_kv_cache=False, image_height=None, image_width=None, + video_num_frames=None, max_batch_size=1, context_parallel_size=1, + task="vision_language_generation", tensor_parallel_size=1, quantization=None, + fp32_layers=(), model_dir=tmp_path, precision="fp16", + max_sequence_length=4096 if tiles > 1 else 768, + verbose=False, backend="trt", family="internvl", + output_path=tmp_path / "model.bundle", graph_transform=None, + ) + writer = BundleWriter(tmp_path / "model.bundle") + staged = {} + add_bytes = writer.add_bytes + + def stage(name, value): + if name.endswith(".plan"): + staged[name] = bytes(value) + add_bytes(name, value) + + monkeypatch.setattr(writer, "add_bytes", stage) + try: + model.build(request, writer) + writer.finish() + finally: + writer.abort() + gc.collect() + assert built == (["prefill", "decode", "vision"] if tiles == 1 else ["prefill", "vision"]) + if tiles > 1: + assert staged["engine.plan"] == staged["prefill.plan"] + assert not live + assert (tmp_path / "model.bundle").is_file() diff --git a/families/internvl/tests/test_prompt_framing.py b/families/internvl/tests/test_prompt_framing.py new file mode 100644 index 0000000000..5192a38963 --- /dev/null +++ b/families/internvl/tests/test_prompt_framing.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compare one-view native framing with actual upstream media insertion.""" + +from types import SimpleNamespace + +import numpy as np +import pytest + +pytest.importorskip("tensorrt") +pytest.importorskip("transformers.models.internvl.processing_internvl") + +from transformers import InternVLProcessor + +from families.internvl.model import _InternVLModel + + +def test_native_prompt_matches_upstream_media_insertion() -> None: + processor = SimpleNamespace( + start_image_token="", end_image_token="", + image_token="", video_token="