From 9d559ba6f1b468d2d6ede52043a8b9b33d74611b Mon Sep 17 00:00:00 2001 From: varunj-msft Date: Tue, 10 Nov 2026 03:56:34 +0000 Subject: [PATCH] Fail a backend scan whose default chat target isn't registered, instead of falling back to one built from environment variables --- pyrit/backend/README.md | 5 +++ .../backend/services/scenario_run_service.py | 19 ++++++--- pyrit/scenario/core/__init__.py | 2 + .../scenario/core/scenario_target_defaults.py | 40 ++++++++++++++++--- .../unit/backend/test_scenario_run_service.py | 22 ++++++++++ .../core/test_scenario_target_defaults.py | 33 +++++++++++++++ 6 files changed, 111 insertions(+), 10 deletions(-) diff --git a/pyrit/backend/README.md b/pyrit/backend/README.md index d0b8ea33ab..aa000a5c15 100644 --- a/pyrit/backend/README.md +++ b/pyrit/backend/README.md @@ -216,6 +216,11 @@ Python-only. Creation does not run an attack. The scenario supplies the objective target and scoring configuration; an omitted adversarial target uses the execution default. +A scan uses the target the server registers as `adversarial_chat` as its default adversarial +target, and the one registered as `objective_scorer_chat` as its default scorer target. When +the scan needs one that isn't registered, for example after a misspelled name in a startup +script, it fails with that reason, usually when it starts (400), instead of using an +`OpenAIChatTarget` built from the `OPENAI_CHAT_*` environment variables. Invalid settings or selector collisions return 400 without registering a factory; invalid request shapes return 422. `all` and `default` are reserved selectors; `types` is also a reserved name. Names cannot replace inherited `ScenarioTechnique` diff --git a/pyrit/backend/services/scenario_run_service.py b/pyrit/backend/services/scenario_run_service.py index ed3b7d5aba..704fa2cdf3 100644 --- a/pyrit/backend/services/scenario_run_service.py +++ b/pyrit/backend/services/scenario_run_service.py @@ -102,7 +102,11 @@ from pyrit.registry import InitializerRegistry, ScenarioRegistry from pyrit.registry.resolution import reject_non_external_params, resolve_declared_params from pyrit.scenario import IncompatibleTechniquePolicy, Scenario -from pyrit.scenario.core import get_default_adversarial_target, override_default_adversarial_target +from pyrit.scenario.core import ( + get_default_adversarial_target, + override_default_adversarial_target, + require_registered_default_targets, +) logger = logging.getLogger(__name__) @@ -405,7 +409,9 @@ def _validate_rerun_references(self, *, stored: ScenarioResult, request: RunScen target = self._configuration_resolver.resolve_adversarial_target( target_name=request.adversarial_target_name ) - target = target if target is not None else get_default_adversarial_target() + if target is None: + with require_registered_default_targets(): + target = get_default_adversarial_target() if target.get_identifier().hash != original_hash: raise ScenarioRunConflictError( "The saved adversarial target changed. Start a new configuration explicitly." @@ -520,7 +526,9 @@ def _capture_launch_references( target = self._configuration_resolver.resolve_adversarial_target( target_name=request.adversarial_target_name ) - target = target if target is not None else get_default_adversarial_target() + if target is None: + with require_registered_default_targets(): + target = get_default_adversarial_target() references["adversarial_target_hash"] = target.get_identifier().hash return references @@ -969,7 +977,8 @@ async def _prepare_run_async( adversarial_target = self._configuration_resolver.resolve_adversarial_target( target_name=request.adversarial_target_name ) - with override_default_adversarial_target(adversarial_target): + # The server's startup scripts register its defaults, so a missing one is an error rather than a fallback. + with override_default_adversarial_target(adversarial_target), require_registered_default_targets(): init_kwargs = await run_legacy_sync_async( self._configuration_resolver.resolve_configuration, scenario_name=request.scenario_name, @@ -1759,7 +1768,7 @@ async def _execute_run_async(self, *, scenario_result_id: str) -> None: handoff_ready = True try: - with override_default_adversarial_target(active.adversarial_target): + with override_default_adversarial_target(active.adversarial_target), require_registered_default_targets(): await active.scenario.run_async() except asyncio.CancelledError: diff --git a/pyrit/scenario/core/__init__.py b/pyrit/scenario/core/__init__.py index 3e1d7e0471..487c82c1e4 100644 --- a/pyrit/scenario/core/__init__.py +++ b/pyrit/scenario/core/__init__.py @@ -36,6 +36,7 @@ get_default_adversarial_target, get_default_scorer_target, override_default_adversarial_target, + require_registered_default_targets, ) from pyrit.scenario.core.scenario_technique import ScenarioTechnique from pyrit.scenario.core.technique_requirements import ( @@ -85,6 +86,7 @@ "get_default_scorer_target": "pyrit.scenario.core.scenario_target_defaults", "get_default_adversarial_target": "pyrit.scenario.core.scenario_target_defaults", "override_default_adversarial_target": "pyrit.scenario.core.scenario_target_defaults", + "require_registered_default_targets": "pyrit.scenario.core.scenario_target_defaults", "parse_technique_token": "pyrit.scenario.core.technique_tokens", "resolve_technique_factories": "pyrit.scenario.core._technique_resolution", "resolve_technique_factories_for_techniques": "pyrit.scenario.core._technique_resolution", diff --git a/pyrit/scenario/core/scenario_target_defaults.py b/pyrit/scenario/core/scenario_target_defaults.py index 43c27e5b29..e890fccfa3 100644 --- a/pyrit/scenario/core/scenario_target_defaults.py +++ b/pyrit/scenario/core/scenario_target_defaults.py @@ -11,6 +11,27 @@ logger = logging.getLogger(__name__) _adversarial_target_override: ContextVar[PromptTarget | None] = ContextVar("adversarial_target_override", default=None) +_registered_defaults_required: ContextVar[bool] = ContextVar("registered_defaults_required", default=False) + + +@contextmanager +def require_registered_default_targets() -> Generator[None, None, None]: + """ + Make a missing default chat target an error in this scope, instead of a plain ``OpenAIChatTarget``. + + Outside the scope, a default that ``TargetRegistry`` doesn't hold falls back to an ``OpenAIChatTarget`` + built from the ``OPENAI_CHAT_*`` environment variables. Code that registers its defaults, such as a + server whose startup scripts do, can wrap scenario work in this scope so that a missing or misspelled + registration fails instead of quietly using that fallback. Scoped and explicit targets are unaffected. + + Yields: + None: While missing defaults are errors. + """ + token = _registered_defaults_required.set(True) + try: + yield + finally: + _registered_defaults_required.reset(token) @contextmanager @@ -67,13 +88,14 @@ def get_default_scorer_target() -> PromptTarget: First checks the ``TargetRegistry`` for an ``"objective_scorer_chat"`` entry (populated by ``TargetInitializer`` from ``OBJECTIVE_SCORER_CHAT_*`` env vars). - Falls back to a plain ``OpenAIChatTarget`` + Falls back to a plain ``OpenAIChatTarget``, except inside ``require_registered_default_targets``. Returns: PromptTarget: The resolved objective scorer chat target. Raises: - ValueError: If the registered target does not support multi-turn. + ValueError: If the registered target does not support multi-turn, or none is registered inside + ``require_registered_default_targets``. """ return _get_default_chat_target(preferred_target_key="objective_scorer_chat") @@ -85,13 +107,15 @@ def get_default_adversarial_target() -> PromptTarget: First checks ``override_default_adversarial_target`` for a scoped default, then the ``TargetRegistry`` for an ``"adversarial_chat"`` entry (populated by ``TargetInitializer`` from ``ADVERSARIAL_CHAT_*`` env vars). - Falls back to a default fallback target with temperature=1.2 + Falls back to a default fallback target with temperature=1.2, except inside + ``require_registered_default_targets``. Returns: PromptTarget: The resolved adversarial chat target. Raises: - ValueError: If the registered target does not support multi-turn. + ValueError: If the registered target does not support multi-turn, or none is registered inside + ``require_registered_default_targets``. """ scoped_target = _adversarial_target_override.get() if scoped_target is not None: @@ -114,7 +138,7 @@ def _get_default_chat_target( Resolution order: 1. ``preferred_target_key`` entry from ``TargetRegistry`` - 2. ``OpenAIChatTarget(...)`` with optional temperature + 2. ``OpenAIChatTarget(...)`` with optional temperature, unless ``require_registered_default_targets`` is active Args: preferred_target_key (str): TargetRegistry key to resolve first. @@ -129,6 +153,7 @@ def _get_default_chat_target( Raises: ValueError: If the resolved target does not satisfy required capabilities. ValueError: If the registry entry exists but is not a PromptTarget. + ValueError: If the registry has no entry while ``require_registered_default_targets`` is active. """ registry = TargetRegistry.get_registry_singleton() target = registry.instances.get(preferred_target_key) @@ -147,6 +172,11 @@ def _get_default_chat_target( return target + if _registered_defaults_required.get(): + raise ValueError( + f"No target is registered as '{preferred_target_key}', so there's no default to use. Register one " + "under that name, for example in a startup script, or choose a target explicitly." + ) logger.warning( f"TargetRegistry entry '{preferred_target_key}' not found. Falling back to default OpenAIChatTarget." ) diff --git a/tests/unit/backend/test_scenario_run_service.py b/tests/unit/backend/test_scenario_run_service.py index 26659b32d3..9db4caa687 100644 --- a/tests/unit/backend/test_scenario_run_service.py +++ b/tests/unit/backend/test_scenario_run_service.py @@ -949,6 +949,28 @@ async def fail_async(*args: object, **kwargs: object) -> None: finally: await service.shutdown_async() + async def test_a_scan_that_needs_an_unregistered_default_target_fails_to_start_async( + self, mock_all_registries: dict[str, Any] + ) -> None: + mock_all_registries["target_registry"].instances.get.side_effect = {"my_target": MockPromptTarget()}.get + service = ScenarioRunService() + + async def initialize_async(*args: object, **kwargs: object) -> Any: + get_default_adversarial_target() + return mock_all_registries["scenario_instance"] + + mock_all_registries["scenario_registry"].create_and_initialize_async.side_effect = initialize_async + try: + with ( + patch("pyrit.scenario.core.scenario_target_defaults.OpenAIChatTarget") as fallback, + pytest.raises(ValueError, match="No target is registered as 'adversarial_chat'"), + ): + await service.start_run_async(request=_make_request()) + fallback.assert_not_called() + assert not service._active_tasks + finally: + await service.shutdown_async() + async def test_cancelled_start_keeps_scope_until_preparation_finishes_async( self, mock_all_registries: dict[str, Any] ) -> None: diff --git a/tests/unit/scenario/core/test_scenario_target_defaults.py b/tests/unit/scenario/core/test_scenario_target_defaults.py index 4e7ee8d2de..8cbe170510 100644 --- a/tests/unit/scenario/core/test_scenario_target_defaults.py +++ b/tests/unit/scenario/core/test_scenario_target_defaults.py @@ -4,6 +4,7 @@ """Scoped adversarial defaults work for framework callers without registry mutation.""" import asyncio +from typing import Any from unittest.mock import MagicMock, patch import pytest @@ -17,6 +18,7 @@ get_default_adversarial_target, get_default_scorer_target, override_default_adversarial_target, + require_registered_default_targets, scenario_target_defaults, ) from pyrit.scenario.scenarios.airt import Scam @@ -139,6 +141,37 @@ def test_openai_fallback_retained_outside_scope(self) -> None: assert get_default_adversarial_target() is fallback create_fallback.assert_called_once_with(temperature=1.2) + @pytest.mark.parametrize( + ("resolve", "name"), + [(get_default_adversarial_target, "adversarial_chat"), (get_default_scorer_target, "objective_scorer_chat")], + ) + def test_a_missing_registration_is_an_error_where_registered_defaults_are_required( + self, resolve: Any, name: str + ) -> None: + registry = MagicMock(spec=TargetRegistry.get_registry_singleton()) + registry.instances.get.return_value = None + fallback = MockPromptTarget() + with ( + patch.object(TargetRegistry, "get_registry_singleton", return_value=registry), + patch.object(scenario_target_defaults, "OpenAIChatTarget", return_value=fallback) as create_fallback, + ): + with require_registered_default_targets(), pytest.raises(ValueError, match=f"registered as '{name}'"): + resolve() + create_fallback.assert_not_called() + assert resolve() is fallback + + def test_registered_and_scoped_targets_are_used_where_registered_defaults_are_required(self) -> None: + registered, scoped = MockPromptTarget(), MockPromptTarget() + registry = MagicMock(spec=TargetRegistry.get_registry_singleton()) + registry.instances.get.return_value = registered + with ( + patch.object(TargetRegistry, "get_registry_singleton", return_value=registry), + require_registered_default_targets(), + ): + assert get_default_scorer_target() is registered + with override_default_adversarial_target(scoped): + assert get_default_adversarial_target() is scoped + def test_scenario_constructor_prefers_explicit_target(self) -> None: scoped, explicit = MockPromptTarget(), MockPromptTarget() scorer = MagicMock(spec=TrueFalseScorer)