[BREAKING] FIX: Fail a backend scan when its default chat target isn't registered

pyrit/backend/README.md

@@ -216,6 +216,11 @@ Python-only.
 
 Creation does not run an attack. The scenario supplies the objective target and
 scoring configuration; an omitted adversarial target uses the execution default.
+A scan uses the target the server registers as `adversarial_chat` as its default adversarial
+target, and the one registered as `objective_scorer_chat` as its default scorer target. When
+the scan needs one that isn't registered, for example after a misspelled name in a startup
+script, it fails with that reason, usually when it starts (400), instead of using an
+`OpenAIChatTarget` built from the `OPENAI_CHAT_*` environment variables.
 Invalid settings or selector collisions return 400 without registering a factory;
 invalid request shapes return 422. `all` and `default` are reserved selectors;
 `types` is also a reserved name. Names cannot replace inherited `ScenarioTechnique`

pyrit/backend/services/scenario_run_service.py

@@ -102,7 +102,11 @@
 from pyrit.registry import InitializerRegistry, ScenarioRegistry
 from pyrit.registry.resolution import reject_non_external_params, resolve_declared_params
 from pyrit.scenario import IncompatibleTechniquePolicy, Scenario
-from pyrit.scenario.core import get_default_adversarial_target, override_default_adversarial_target
+from pyrit.scenario.core import (
+    get_default_adversarial_target,
+    override_default_adversarial_target,
+    require_registered_default_targets,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -405,7 +409,9 @@ def _validate_rerun_references(self, *, stored: ScenarioResult, request: RunScen
             target = self._configuration_resolver.resolve_adversarial_target(
                 target_name=request.adversarial_target_name
             )
-            target = target if target is not None else get_default_adversarial_target()
+            if target is None:
+                with require_registered_default_targets():
+                    target = get_default_adversarial_target()
             if target.get_identifier().hash != original_hash:
                 raise ScenarioRunConflictError(
                     "The saved adversarial target changed. Start a new configuration explicitly."
@@ -520,7 +526,9 @@ def _capture_launch_references(
             target = self._configuration_resolver.resolve_adversarial_target(
                 target_name=request.adversarial_target_name
             )
-            target = target if target is not None else get_default_adversarial_target()
+            if target is None:
+                with require_registered_default_targets():
+                    target = get_default_adversarial_target()
             references["adversarial_target_hash"] = target.get_identifier().hash
         return references
 
@@ -969,7 +977,8 @@ async def _prepare_run_async(
         adversarial_target = self._configuration_resolver.resolve_adversarial_target(
             target_name=request.adversarial_target_name
         )
-        with override_default_adversarial_target(adversarial_target):
+        # The server's startup scripts register its defaults, so a missing one is an error rather than a fallback.
+        with override_default_adversarial_target(adversarial_target), require_registered_default_targets():
             init_kwargs = await run_legacy_sync_async(
                 self._configuration_resolver.resolve_configuration,
                 scenario_name=request.scenario_name,
@@ -1759,7 +1768,7 @@ async def _execute_run_async(self, *, scenario_result_id: str) -> None:
         handoff_ready = True
 
         try:
-            with override_default_adversarial_target(active.adversarial_target):
+            with override_default_adversarial_target(active.adversarial_target), require_registered_default_targets():
                 await active.scenario.run_async()
 
         except asyncio.CancelledError:

pyrit/scenario/core/__init__.py

@@ -36,6 +36,7 @@
         get_default_adversarial_target,
         get_default_scorer_target,
         override_default_adversarial_target,
+        require_registered_default_targets,
     )
     from pyrit.scenario.core.scenario_technique import ScenarioTechnique
     from pyrit.scenario.core.technique_requirements import (
@@ -85,6 +86,7 @@
     "get_default_scorer_target": "pyrit.scenario.core.scenario_target_defaults",
     "get_default_adversarial_target": "pyrit.scenario.core.scenario_target_defaults",
     "override_default_adversarial_target": "pyrit.scenario.core.scenario_target_defaults",
+    "require_registered_default_targets": "pyrit.scenario.core.scenario_target_defaults",
     "parse_technique_token": "pyrit.scenario.core.technique_tokens",
     "resolve_technique_factories": "pyrit.scenario.core._technique_resolution",
     "resolve_technique_factories_for_techniques": "pyrit.scenario.core._technique_resolution",

pyrit/scenario/core/scenario_target_defaults.py

@@ -11,6 +11,27 @@
 
 logger = logging.getLogger(__name__)
 _adversarial_target_override: ContextVar[PromptTarget | None] = ContextVar("adversarial_target_override", default=None)
+_registered_defaults_required: ContextVar[bool] = ContextVar("registered_defaults_required", default=False)
+
+
+@contextmanager
+def require_registered_default_targets() -> Generator[None, None, None]:
+    """
+    Make a missing default chat target an error in this scope, instead of a plain ``OpenAIChatTarget``.
+
+    Outside the scope, a default that ``TargetRegistry`` doesn't hold falls back to an ``OpenAIChatTarget``
+    built from the ``OPENAI_CHAT_*`` environment variables. Code that registers its defaults, such as a
+    server whose startup scripts do, can wrap scenario work in this scope so that a missing or misspelled
+    registration fails instead of quietly using that fallback. Scoped and explicit targets are unaffected.
+
+    Yields:
+        None: While missing defaults are errors.
+    """
+    token = _registered_defaults_required.set(True)
+    try:
+        yield
+    finally:
+        _registered_defaults_required.reset(token)
 
 
 @contextmanager
@@ -67,13 +88,14 @@ def get_default_scorer_target() -> PromptTarget:
 
     First checks the ``TargetRegistry`` for an ``"objective_scorer_chat"`` entry
     (populated by ``TargetInitializer`` from ``OBJECTIVE_SCORER_CHAT_*`` env vars).
-    Falls back to a plain ``OpenAIChatTarget``
+    Falls back to a plain ``OpenAIChatTarget``, except inside ``require_registered_default_targets``.
 
     Returns:
         PromptTarget: The resolved objective scorer chat target.
 
     Raises:
-        ValueError: If the registered target does not support multi-turn.
+        ValueError: If the registered target does not support multi-turn, or none is registered inside
+            ``require_registered_default_targets``.
     """
     return _get_default_chat_target(preferred_target_key="objective_scorer_chat")
 
@@ -85,13 +107,15 @@ def get_default_adversarial_target() -> PromptTarget:
     First checks ``override_default_adversarial_target`` for a scoped default,
     then the ``TargetRegistry`` for an ``"adversarial_chat"`` entry
     (populated by ``TargetInitializer`` from ``ADVERSARIAL_CHAT_*`` env vars).
-    Falls back to a default fallback target with temperature=1.2
+    Falls back to a default fallback target with temperature=1.2, except inside
+    ``require_registered_default_targets``.
 
     Returns:
         PromptTarget: The resolved adversarial chat target.
 
     Raises:
-        ValueError: If the registered target does not support multi-turn.
+        ValueError: If the registered target does not support multi-turn, or none is registered inside
+            ``require_registered_default_targets``.
     """
     scoped_target = _adversarial_target_override.get()
     if scoped_target is not None:
@@ -114,7 +138,7 @@ def _get_default_chat_target(
 
     Resolution order:
     1. ``preferred_target_key`` entry from ``TargetRegistry``
-    2. ``OpenAIChatTarget(...)`` with optional temperature
+    2. ``OpenAIChatTarget(...)`` with optional temperature, unless ``require_registered_default_targets`` is active
 
     Args:
         preferred_target_key (str): TargetRegistry key to resolve first.
@@ -129,6 +153,7 @@ def _get_default_chat_target(
     Raises:
         ValueError: If the resolved target does not satisfy required capabilities.
         ValueError: If the registry entry exists but is not a PromptTarget.
+        ValueError: If the registry has no entry while ``require_registered_default_targets`` is active.
     """
     registry = TargetRegistry.get_registry_singleton()
     target = registry.instances.get(preferred_target_key)
@@ -147,6 +172,11 @@ def _get_default_chat_target(
 
         return target
 
+    if _registered_defaults_required.get():
+        raise ValueError(
+            f"No target is registered as '{preferred_target_key}', so there's no default to use. Register one "
+            "under that name, for example in a startup script, or choose a target explicitly."
+        )
     logger.warning(
         f"TargetRegistry entry '{preferred_target_key}' not found. Falling back to default OpenAIChatTarget."
     )

tests/unit/backend/test_scenario_run_service.py

@@ -949,6 +949,28 @@ async def fail_async(*args: object, **kwargs: object) -> None:
         finally:
             await service.shutdown_async()
 
+    async def test_a_scan_that_needs_an_unregistered_default_target_fails_to_start_async(
+        self, mock_all_registries: dict[str, Any]
+    ) -> None:
+        mock_all_registries["target_registry"].instances.get.side_effect = {"my_target": MockPromptTarget()}.get
+        service = ScenarioRunService()
+
+        async def initialize_async(*args: object, **kwargs: object) -> Any:
+            get_default_adversarial_target()
+            return mock_all_registries["scenario_instance"]
+
+        mock_all_registries["scenario_registry"].create_and_initialize_async.side_effect = initialize_async
+        try:
+            with (
+                patch("pyrit.scenario.core.scenario_target_defaults.OpenAIChatTarget") as fallback,
+                pytest.raises(ValueError, match="No target is registered as 'adversarial_chat'"),
+            ):
+                await service.start_run_async(request=_make_request())
+            fallback.assert_not_called()
+            assert not service._active_tasks
+        finally:
+            await service.shutdown_async()
+
     async def test_cancelled_start_keeps_scope_until_preparation_finishes_async(
         self, mock_all_registries: dict[str, Any]
     ) -> None:

tests/unit/scenario/core/test_scenario_target_defaults.py

@@ -4,6 +4,7 @@
 """Scoped adversarial defaults work for framework callers without registry mutation."""
 
 import asyncio
+from typing import Any
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -17,6 +18,7 @@
     get_default_adversarial_target,
     get_default_scorer_target,
     override_default_adversarial_target,
+    require_registered_default_targets,
     scenario_target_defaults,
 )
 from pyrit.scenario.scenarios.airt import Scam
@@ -139,6 +141,37 @@ def test_openai_fallback_retained_outside_scope(self) -> None:
             assert get_default_adversarial_target() is fallback
             create_fallback.assert_called_once_with(temperature=1.2)
 
+    @pytest.mark.parametrize(
+        ("resolve", "name"),
+        [(get_default_adversarial_target, "adversarial_chat"), (get_default_scorer_target, "objective_scorer_chat")],
+    )
+    def test_a_missing_registration_is_an_error_where_registered_defaults_are_required(
+        self, resolve: Any, name: str
+    ) -> None:
+        registry = MagicMock(spec=TargetRegistry.get_registry_singleton())
+        registry.instances.get.return_value = None
+        fallback = MockPromptTarget()
+        with (
+            patch.object(TargetRegistry, "get_registry_singleton", return_value=registry),
+            patch.object(scenario_target_defaults, "OpenAIChatTarget", return_value=fallback) as create_fallback,
+        ):
+            with require_registered_default_targets(), pytest.raises(ValueError, match=f"registered as '{name}'"):
+                resolve()
+            create_fallback.assert_not_called()
+            assert resolve() is fallback
+
+    def test_registered_and_scoped_targets_are_used_where_registered_defaults_are_required(self) -> None:
+        registered, scoped = MockPromptTarget(), MockPromptTarget()
+        registry = MagicMock(spec=TargetRegistry.get_registry_singleton())
+        registry.instances.get.return_value = registered
+        with (
+            patch.object(TargetRegistry, "get_registry_singleton", return_value=registry),
+            require_registered_default_targets(),
+        ):
+            assert get_default_scorer_target() is registered
+            with override_default_adversarial_target(scoped):
+                assert get_default_adversarial_target() is scoped
+
     def test_scenario_constructor_prefers_explicit_target(self) -> None:
         scoped, explicit = MockPromptTarget(), MockPromptTarget()
         scorer = MagicMock(spec=TrueFalseScorer)