diff --git a/.pyrit_conf_example b/.pyrit_conf_example index 51488ee0d8..f31a5feaec 100644 --- a/.pyrit_conf_example +++ b/.pyrit_conf_example @@ -161,6 +161,14 @@ allow_custom_initializers: false # as scenario_presets_source. Defaults to ~/.pyrit/scenario_suites. # scenario_suites_source: https://account.blob.core.windows.net/container/scenario_suites +# Optional storage for saved targets, converters, scorers, and techniques, so they +# come back when the backend restarts. Each kind has its own folder under the source. +# This supports the same local directory and Azure Blob sources as +# scenario_presets_source. If a saved entry can't be restored, the backend still +# starts, lists the failure in its runtime status, and refuses to use that entry's +# name until the file is fixed or deleted. Defaults to ~/.pyrit/registry_recipes. +# registry_recipes_source: https://account.blob.core.windows.net/container/registry_recipes + # Upload Folder for API-Created Targets # ------------------------------------- # Folder that targets created through the REST API may upload local files from diff --git a/doc/getting_started/pyrit_conf.md b/doc/getting_started/pyrit_conf.md index bba64b45d4..feb96f07f4 100644 --- a/doc/getting_started/pyrit_conf.md +++ b/doc/getting_started/pyrit_conf.md @@ -194,6 +194,18 @@ Suites store ordered references to scenario presets and resolve each preset's cu A SAS query string may be included; otherwise, PyRIT uses `DefaultAzureCredential`. The default is `~/.pyrit/scenario_suites`. Local writes use the same hidden lock-file and atomic replacement behavior as scenario presets. Startup scans the configured source so malformed suite documents are reported immediately without preventing the backend from starting. +### `registry_recipes_source` + +Stores saved targets, converters, scorers, and techniques in a local directory or Azure Blob container, so they come back when the backend restarts. An Azure URI may include a blob-name prefix: + +```yaml +registry_recipes_source: https://account.blob.core.windows.net/pyrit-storage/registry-recipes +``` + +Each kind has its own folder under the source (`targets/`, `converters/`, `scorers/`, `techniques/`), with one JSON file per saved entry named after the entry. A file holds what's needed to rebuild the entry; restore refuses a file that stores a value for a parameter its class marks as a credential, such as `api_key`. A SAS query string may be included; otherwise, PyRIT uses `DefaultAzureCredential`. The default is `~/.pyrit/registry_recipes`. + +The backend restores saved entries at startup and after a live apply, once the configured initializers have registered theirs. If a saved entry can't be restored, the backend still becomes ready and lists each failure with its reason in `GET /api/config/runtime`. The failed entry keeps its name: looking it up, or starting a run that uses it, fails with the reason (409) instead of finding nothing or something else, and nothing else can take the name. While a saved technique is broken, a launch has to name its techniques, because choosing them by `all`, `default`, or a tag could quietly leave it out. If a whole folder can't be listed, nothing is restored and every name of that kind that isn't registered is held the same way. Fix or delete the listed files, then apply or restart. Files whose names aren't valid entry names (up to 64 lowercase letters, digits, and underscores, starting with a letter) are listed as ignored. + ### `initialization_scripts` Local paths to custom Python scripts containing `PyRITInitializer` subclasses. Paths can be absolute or relative to the current working directory. diff --git a/frontend/e2e/fixtures/manual_send_backend.py b/frontend/e2e/fixtures/manual_send_backend.py index 0c46ed2398..8be9b44941 100644 --- a/frontend/e2e/fixtures/manual_send_backend.py +++ b/frontend/e2e/fixtures/manual_send_backend.py @@ -4,6 +4,7 @@ """Offline backend for manual-send browser tests, with no real configuration or providers.""" import os +import tempfile from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from unittest.mock import AsyncMock, patch @@ -27,19 +28,22 @@ async def _initialize_offline_async(self: ConfigurationLoader, *, raise_on_initi @asynccontextmanager async def _offline_lifespan_async(application: FastAPI) -> AsyncGenerator[None, None]: - config = ConfigurationLoader(memory_db_type="in_memory", env_files=[], env_akv_ref=[]) - with ( - patch.dict(os.environ, {"PYRIT_AUTH_MODE": "local", "PYRIT_DEV_MODE": "true"}, clear=True), - patch.object( - ConfigurationFileService, - "_read_source_async", - new=AsyncMock(return_value="memory_db_type: in_memory\nenv_files: []\ninitializers: []\n"), - ), - patch.object(ConfigurationLoader, "load_with_overrides", return_value=config), - patch.object(ConfigurationLoader, "initialize_pyrit_async", new=_initialize_offline_async), - ): - async with lifespan(application): - yield + with tempfile.TemporaryDirectory() as recipes: + config = ConfigurationLoader( + memory_db_type="in_memory", env_files=[], env_akv_ref=[], registry_recipes_source=recipes + ) + with ( + patch.dict(os.environ, {"PYRIT_AUTH_MODE": "local", "PYRIT_DEV_MODE": "true"}, clear=True), + patch.object( + ConfigurationFileService, + "_read_source_async", + new=AsyncMock(return_value="memory_db_type: in_memory\nenv_files: []\ninitializers: []\n"), + ), + patch.object(ConfigurationLoader, "load_with_overrides", return_value=config), + patch.object(ConfigurationLoader, "initialize_pyrit_async", new=_initialize_offline_async), + ): + async with lifespan(application): + yield app.router.lifespan_context = _offline_lifespan_async diff --git a/pyrit/backend/README.md b/pyrit/backend/README.md index d0b8ea33ab..47a9427ab8 100644 --- a/pyrit/backend/README.md +++ b/pyrit/backend/README.md @@ -337,8 +337,9 @@ The chat preserves failed drafts across replacement, but requires converter choi to be reviewed again rather than restoring outputs tied to the previous registry. The offline browser fixture `frontend.e2e.fixtures.manual_send_backend:app` uses the real -backend lifecycle with isolated in-memory SQLite, no environment files, and no default -providers. The recovery tests register deterministic loopback targets. For example, start +backend lifecycle with isolated in-memory SQLite, no environment files, no saved registry +entries, and no default providers. The recovery tests register deterministic loopback +targets. For example, start it with `uv run python -m uvicorn frontend.e2e.fixtures.manual_send_backend:app --host 127.0.0.1 --port 18213` (set `PYTHONUTF8=1` on Windows). In another shell, set `PYRIT_BACKEND_URL=http://127.0.0.1:18213` and `E2E_FRONTEND_PORT=31213`, then run `npx playwright test chat-recovery --project seeded --workers 1` diff --git a/pyrit/backend/middleware/error_handlers.py b/pyrit/backend/middleware/error_handlers.py index 5973d6289e..9186f41531 100644 --- a/pyrit/backend/middleware/error_handlers.py +++ b/pyrit/backend/middleware/error_handlers.py @@ -11,9 +11,12 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse +from pyrit.backend.middleware.auth import has_admin_access from pyrit.backend.middleware.request_size import RequestTooLargeError, request_too_large_response from pyrit.backend.models import FieldError, ProblemDetail from pyrit.exceptions.analytics_exception import AnalyticsException +from pyrit.registry import SavedEntryUnavailableError, TechniqueSelectionUnavailableError +from pyrit.registry.file_document_storage import DocumentStorageUnavailableError, DocumentUnreadableError logger = logging.getLogger(__name__) @@ -169,6 +172,68 @@ async def permission_error_handler( # pyrit-async-suffix-exempt content=problem.model_dump(exclude_none=True), ) + @app.exception_handler(DocumentStorageUnavailableError) + async def storage_unavailable_handler( # pyrit-async-suffix-exempt + request: Request, + exc: DocumentStorageUnavailableError, + ) -> JSONResponse: + """ + Handle stored documents that can't be read or written as 503 Service Unavailable. + + The log names the storage location, without credentials, and the reason, with the + traceback at debug level. The response never carries a path or URL. + + Returns: + JSONResponse: RFC 7807 problem detail response with 503 status. + """ + logger.warning(f"{exc} ({request.method} {request.url.path})") + logger.debug("Storage error details", exc_info=exc) + if isinstance(exc, DocumentUnreadableError): + detail = f"{exc.label} '{exc.name}' can't be read: {exc.reason}" + else: + detail = f"{exc.label} storage is unavailable" + problem = ProblemDetail( + type="/errors/storage-unavailable", + title="Service Unavailable", + status=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=detail, + instance=str(request.url.path), + ) + + return JSONResponse( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + content=problem.model_dump(exclude_none=True), + ) + + @app.exception_handler(SavedEntryUnavailableError) + async def saved_entry_unavailable_handler( # pyrit-async-suffix-exempt + request: Request, + exc: SavedEntryUnavailableError, + ) -> JSONResponse: + """ + Handle a name held for a saved entry that couldn't be restored as 409 Conflict. + + Administrators get the full message; anyone else gets a fixed explanation of the reason, + since the message can quote a server setting. + + Returns: + JSONResponse: RFC 7807 problem detail response with 409 status. + """ + return _conflict(request=request, detail=exc.describe(with_details=_shows_details(request))) + + @app.exception_handler(TechniqueSelectionUnavailableError) + async def technique_selection_unavailable_handler( # pyrit-async-suffix-exempt + request: Request, + exc: TechniqueSelectionUnavailableError, + ) -> JSONResponse: + """ + Handle a broad technique selection made while a saved technique couldn't be restored as 409 Conflict. + + Returns: + JSONResponse: RFC 7807 problem detail response with 409 status. + """ + return _conflict(request=request, detail=str(exc)) + @app.exception_handler(NotImplementedError) async def not_implemented_handler( # pyrit-async-suffix-exempt request: Request, @@ -223,3 +288,33 @@ async def generic_exception_handler( # pyrit-async-suffix-exempt status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content=problem.model_dump(exclude_none=True), ) + + +def _conflict(*, request: Request, detail: str) -> JSONResponse: + """ + Build a 409 Conflict problem response. + + Returns: + JSONResponse: RFC 7807 problem detail response with 409 status. + """ + problem = ProblemDetail( + type="/errors/conflict", + title="Conflict", + status=status.HTTP_409_CONFLICT, + detail=detail, + instance=str(request.url.path), + ) + return JSONResponse(status_code=status.HTTP_409_CONFLICT, content=problem.model_dump(exclude_none=True)) + + +def _shows_details(request: Request) -> bool: + """ + Return whether the caller may see a restore failure's full message, which can quote a server setting. + + Returns: + bool: True for an administrator; False otherwise, including when no authentication policy is set. + """ + try: + return has_admin_access(request) + except RuntimeError: + return False diff --git a/pyrit/backend/routes/attacks.py b/pyrit/backend/routes/attacks.py index 51cc1d4dc0..76bac6db9f 100644 --- a/pyrit/backend/routes/attacks.py +++ b/pyrit/backend/routes/attacks.py @@ -45,6 +45,7 @@ from pyrit.backend.services.media_persistence import MediaAccessDeniedError from pyrit.common import print_deprecation_message from pyrit.memory.memory_interface import AttackStateConflictError +from pyrit.registry import SavedEntryUnavailableError logger = logging.getLogger(__name__) @@ -666,6 +667,8 @@ async def add_message( # pyrit-async-suffix-exempt status_code=status.HTTP_400_BAD_REQUEST, detail=error_msg, ) from e + except SavedEntryUnavailableError: + raise except Exception as e: logger.exception("Failed to add message to attack '%s'", attack_result_id) raise HTTPException( diff --git a/pyrit/backend/routes/converters.py b/pyrit/backend/routes/converters.py index 10d129394e..d97841170e 100644 --- a/pyrit/backend/routes/converters.py +++ b/pyrit/backend/routes/converters.py @@ -24,6 +24,7 @@ UnregisteredConverter, ) from pyrit.backend.services import get_converter_service +from pyrit.registry import SavedEntryUnavailableError router = APIRouter(prefix="/converters", tags=["converters"]) @@ -89,6 +90,8 @@ async def create_converter( status_code=status.HTTP_400_BAD_REQUEST, detail=str(e), ) from e + except SavedEntryUnavailableError: + raise except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -165,6 +168,8 @@ async def preview_conversion(request: ConverterPreviewRequest) -> ConverterPrevi status_code=status.HTTP_400_BAD_REQUEST, detail=str(e), ) from e + except SavedEntryUnavailableError: + raise except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/pyrit/backend/routes/initializers.py b/pyrit/backend/routes/initializers.py index 50fa79db85..529f92d641 100644 --- a/pyrit/backend/routes/initializers.py +++ b/pyrit/backend/routes/initializers.py @@ -14,7 +14,6 @@ DELETE /api/initializers/{name} — unregister an initializer """ -from azure.core.exceptions import AzureError from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from pyrit.backend.middleware.auth import require_admin @@ -35,19 +34,6 @@ router = APIRouter(prefix="/initializers", tags=["initializers"]) -def _custom_storage_unavailable() -> HTTPException: - """ - Create a sanitized response for unavailable custom initializer storage. - - Returns: - HTTPException: A service-unavailable response without SDK details. - """ - return HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Custom initializer storage is temporarily unavailable", - ) - - def _configured_initializers(request: Request) -> list[ConfiguredInitializerSetting]: """ Read the initializer list captured from ``.pyrit_conf`` at backend startup. @@ -138,10 +124,7 @@ async def list_custom_initializers(request: Request) -> CustomInitializerListRes CustomInitializerListResponse: The configured source and stored scripts. """ _check_custom_initializers_allowed(request) - try: - return await get_initializer_service().list_custom_initializers_async() - except AzureError as exc: - raise _custom_storage_unavailable() from exc + return await get_initializer_service().list_custom_initializers_async() @router.get( @@ -205,8 +188,6 @@ async def register_initializer( # pyrit-async-suffix-exempt try: return await service.register_initializer_async(name=body.name, script_content=body.script_content) - except AzureError as exc: - raise _custom_storage_unavailable() from exc except ValueError as e: detail = str(e) if "already registered" in detail: @@ -243,8 +224,6 @@ async def unregister_initializer( # pyrit-async-suffix-exempt try: await service.unregister_initializer_async(initializer_name=initializer_name) - except AzureError as exc: - raise _custom_storage_unavailable() from exc except ValueError as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, diff --git a/pyrit/backend/routes/message_sends.py b/pyrit/backend/routes/message_sends.py index 297783c99e..7441f28af5 100644 --- a/pyrit/backend/routes/message_sends.py +++ b/pyrit/backend/routes/message_sends.py @@ -15,6 +15,7 @@ from pyrit.backend.services.manual_send_scheduler import ManualSendConflictError, ManualSendQueueFullError from pyrit.backend.services.message_send_service import MessageSendNotFoundError, get_message_send_service from pyrit.memory.memory_interface import AttackStateConflictError +from pyrit.registry import SavedEntryUnavailableError logger = logging.getLogger(__name__) router = APIRouter(prefix="/attacks", tags=["attacks"]) @@ -49,6 +50,8 @@ async def submit_message_send_async( except ValueError as exc: is_not_found = "not found" in str(exc).lower() or "does not exist" in str(exc).lower() raise HTTPException(status_code=404 if is_not_found else 400, detail=str(exc)) from exc + except SavedEntryUnavailableError: + raise except Exception as exc: logger.exception("Failed to accept message send for attack '%s'", attack_result_id) raise HTTPException(status_code=500, detail="Could not accept the message send. Check server logs.") from exc diff --git a/pyrit/backend/routes/scenario_presets.py b/pyrit/backend/routes/scenario_presets.py index 4b3fc1db00..973112df7b 100644 --- a/pyrit/backend/routes/scenario_presets.py +++ b/pyrit/backend/routes/scenario_presets.py @@ -19,7 +19,6 @@ from typing import Annotated -from azure.core.exceptions import AzureError from fastapi import APIRouter, Depends, HTTPException, Query, status from pyrit.backend.middleware.auth import current_user_name, require_admin @@ -38,19 +37,6 @@ router = APIRouter(prefix="/scenario-presets", tags=["scenario-presets"]) -def _storage_unavailable() -> HTTPException: - """ - Create a sanitized response for unavailable preset storage. - - Returns: - HTTPException: A service-unavailable response without SDK details. - """ - return HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Scenario preset storage is temporarily unavailable", - ) - - def _not_found(name: str) -> HTTPException: """ Create a not-found response for a missing preset. @@ -79,8 +65,6 @@ async def _load_preset_or_404_async(name: str) -> ScenarioPresetResponse: """ try: preset = await get_scenario_preset_service().get_preset_async(name=name) - except AzureError as exc: - raise _storage_unavailable() from exc except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None @@ -106,10 +90,7 @@ async def list_scenario_presets( # pyrit-async-suffix-exempt Returns: ScenarioPresetListResponse: The configured source and the stored presets. """ - try: - return await get_scenario_preset_service().list_presets_async(include_estimates=include_estimates) - except AzureError as exc: - raise _storage_unavailable() from exc + return await get_scenario_preset_service().list_presets_async(include_estimates=include_estimates) @router.get( @@ -177,8 +158,6 @@ async def create_scenario_preset( # pyrit-async-suffix-exempt status_code=status.HTTP_409_CONFLICT, detail=f"Scenario preset '{preset.name}' already exists", ) from exc - except AzureError as exc: - raise _storage_unavailable() from exc @router.put( @@ -239,8 +218,6 @@ async def update_scenario_preset( # pyrit-async-suffix-exempt status_code=status.HTTP_409_CONFLICT, detail=f"Scenario preset '{name}' changed since it was read; re-read it and retry", ) from exc - except AzureError as exc: - raise _storage_unavailable() from exc @router.delete( @@ -261,8 +238,6 @@ async def delete_scenario_preset(name: IdentifierStr) -> None: # pyrit-async-su """ try: deleted = await get_scenario_preset_service().delete_preset_async(name=name) - except AzureError as exc: - raise _storage_unavailable() from exc except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None @@ -312,8 +287,6 @@ async def resolve_scenario_preset( # pyrit-async-suffix-exempt status_code=status.HTTP_409_CONFLICT, detail=f"Scenario preset '{name}' changed since it was read; re-read it and retry", ) from None - except AzureError as exc: - raise _storage_unavailable() from exc except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None diff --git a/pyrit/backend/routes/scenario_suites.py b/pyrit/backend/routes/scenario_suites.py index 0b4fa7e09a..f1ec16872e 100644 --- a/pyrit/backend/routes/scenario_suites.py +++ b/pyrit/backend/routes/scenario_suites.py @@ -6,7 +6,6 @@ from typing import Annotated from uuid import UUID -from azure.core.exceptions import AzureError from fastapi import APIRouter, Depends, HTTPException, Request, status from pyrit.backend.middleware.auth import current_user_name, require_admin @@ -31,19 +30,6 @@ router = APIRouter(prefix="/scenario-suites", tags=["scenario-suites"]) -def _storage_unavailable() -> HTTPException: - """ - Create a sanitized response for unavailable suite storage. - - Returns: - HTTPException: A service-unavailable response without SDK details. - """ - return HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Scenario suite storage is temporarily unavailable", - ) - - def _not_found(name: str) -> HTTPException: """ Create a not-found response for a missing suite. @@ -66,8 +52,6 @@ async def _load_suite_or_404_async(name: str) -> ScenarioSuiteResponse: """ try: suite = await get_scenario_suite_service().get_suite_async(name=name) - except AzureError as exc: - raise _storage_unavailable() from exc except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None @@ -84,10 +68,7 @@ async def list_scenario_suites() -> ScenarioSuiteListResponse: # pyrit-async-su Returns: ScenarioSuiteListResponse: Suites from both origins with current entry statuses. """ - try: - return await get_scenario_suite_service().list_suites_async() - except AzureError as exc: - raise _storage_unavailable() from exc + return await get_scenario_suite_service().list_suites_async() @router.get( @@ -181,8 +162,6 @@ async def launch_scenario_suite( # pyrit-async-suffix-exempt raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None except ScenarioSuiteExecutionCapacityError as exc: raise HTTPException(status_code=status.HTTP_429_TOO_MANY_REQUESTS, detail=str(exc)) from None - except AzureError as exc: - raise _storage_unavailable() from exc except RuntimeError as exc: raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc)) from None @@ -239,8 +218,6 @@ async def create_scenario_suite( # pyrit-async-suffix-exempt status_code=status.HTTP_409_CONFLICT, detail=f"Scenario suite '{suite.name}' already exists", ) from exc - except AzureError as exc: - raise _storage_unavailable() from exc @router.put( @@ -289,8 +266,6 @@ async def update_scenario_suite( # pyrit-async-suffix-exempt status_code=status.HTTP_409_CONFLICT, detail=f"Scenario suite '{name}' changed since it was read; re-read it and retry", ) from exc - except AzureError as exc: - raise _storage_unavailable() from exc @router.delete( @@ -303,8 +278,6 @@ async def delete_scenario_suite(name: IdentifierStr) -> None: # pyrit-async-suf """Delete one stored scenario suite.""" try: deleted = await get_scenario_suite_service().delete_suite_async(name=name) - except AzureError as exc: - raise _storage_unavailable() from exc except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None diff --git a/pyrit/backend/routes/scenarios.py b/pyrit/backend/routes/scenarios.py index f7cc97bba4..945f5a3834 100644 --- a/pyrit/backend/routes/scenarios.py +++ b/pyrit/backend/routes/scenarios.py @@ -220,7 +220,10 @@ async def load_scenario_datasets_async( responses={ 400: {"model": ProblemDetail, "description": "Invalid request (bad scenario/target/technique)"}, 404: {"model": ProblemDetail, "description": "Saved run not found"}, - 409: {"model": ProblemDetail, "description": "Saved run cannot be resumed"}, + 409: { + "model": ProblemDetail, + "description": "Saved run cannot be resumed, or a launch script would replace a saved entry", + }, }, ) async def start_scenario_run( # pyrit-async-suffix-exempt diff --git a/pyrit/backend/routes/scorers.py b/pyrit/backend/routes/scorers.py index 445f2830b3..65287aea6e 100644 --- a/pyrit/backend/routes/scorers.py +++ b/pyrit/backend/routes/scorers.py @@ -17,6 +17,7 @@ ) from pyrit.backend.services.scorer_service import get_scorer_service from pyrit.models.catalog import ScorerInstance +from pyrit.registry import SavedEntryUnavailableError router = APIRouter(prefix="/scorers", tags=["scorers"]) @@ -65,6 +66,8 @@ async def create_scorer( return await get_scorer_service().create_scorer_async(request=request) except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + except SavedEntryUnavailableError: + raise except Exception as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/pyrit/backend/routes/targets.py b/pyrit/backend/routes/targets.py index c520a42e05..a10164b4bc 100644 --- a/pyrit/backend/routes/targets.py +++ b/pyrit/backend/routes/targets.py @@ -23,6 +23,7 @@ ) from pyrit.backend.services import get_target_service from pyrit.models.catalog import TargetInstance +from pyrit.registry import SavedEntryUnavailableError router = APIRouter(prefix="/targets", tags=["targets"]) @@ -132,6 +133,8 @@ async def create_target( status_code=status.HTTP_400_BAD_REQUEST, detail=str(e), ) from e + except SavedEntryUnavailableError: + raise except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/pyrit/backend/services/runtime_lifecycle.py b/pyrit/backend/services/runtime_lifecycle.py index daef21f0ed..c6ae1be279 100644 --- a/pyrit/backend/services/runtime_lifecycle.py +++ b/pyrit/backend/services/runtime_lifecycle.py @@ -7,7 +7,7 @@ import logging import os import uuid -from typing import Any +from typing import TYPE_CHECKING, Any from fastapi import FastAPI @@ -26,10 +26,14 @@ from pyrit.common.path import CONFIGURATION_DIRECTORY_PATH from pyrit.memory import CentralMemory from pyrit.registry import InitializerRegistry, ScenarioPresetRegistry, ScenarioSuiteRegistry +from pyrit.registry.registry_recipe_restore import RegistryRecipeRestorer from pyrit.setup import ConfigurationLoader from pyrit.setup.environment_loading import resolve_environment_async from pyrit.setup.initialization import validate_reinitialization_memory +if TYPE_CHECKING: + from pyrit.models.catalog import IgnoredRecipeFile, RestoreFailure + logger = logging.getLogger(__name__) @@ -46,6 +50,8 @@ def __init__(self, *, app: FastAPI, source: ConfigurationFileService) -> None: self.version: str | None = None self.outcome = "starting" self.message = "" + self.restore_failures: list[RestoreFailure] = [] + self.restore_ignored: list[IgnoredRecipeFile] = [] self.operations: set[asyncio.Task[None]] = set() self.management_operations: set[asyncio.Task[None]] = set() self.apply_task: asyncio.Task[None] | None = None @@ -75,6 +81,8 @@ def status(self) -> dict[str, Any]: "message": self.message, "enabled": self.topology_supported, "applying": self.apply_task is not None and not self.apply_task.done(), + "restore_failures": [failure.model_dump(mode="json") for failure in self.restore_failures], + "restore_ignored": [file.model_dump(mode="json") for file in self.restore_ignored], } async def _load_async(self) -> ConfigurationLoader: @@ -119,6 +127,32 @@ def _publish(self, config: ConfigurationLoader) -> None: self.generation = str(uuid.uuid4()) self.state, self.outcome, self.message = "ready", "success", "PyRIT is ready." + async def _restore_saved_entries_async(self, config: ConfigurationLoader) -> None: + """ + Rebuild saved registry entries after the startup scripts have registered theirs. + + The runtime becomes ready even when some can't be restored. Each one that failed keeps its name, + so looking it up or starting a run that uses it fails with the reason instead of finding nothing + or something else, and the failures are listed in the runtime status. + + Args: + config (ConfigurationLoader): The loaded configuration naming the storage source. + """ + restorer = RegistryRecipeRestorer(source=config.registry_recipes_source) + # Reading can wait on storage, so it runs in a worker thread. Entries are built on this loop, as + # the target API builds them, because a constructor such as HuggingFaceChatTarget's starts a task + # on it. A slow constructor holds the loop while it runs. + saved = await asyncio.to_thread(restorer.read) + report = restorer.restore(saved=saved) + self.restore_failures, self.restore_ignored = report.failures, report.ignored + if report.failures: + count = len(report.failures) + logger.warning( + "%d saved registry %s couldn't be restored. Fix or delete them; see GET /api/config/runtime.", + count, + "entry" if count == 1 else "entries", + ) + async def startup_async(self) -> None: """Keep management alive even when parsing or initialization fails.""" self.app.state.allow_custom_initializers = False @@ -137,6 +171,7 @@ async def startup_async(self) -> None: await config.initialize_pyrit_async(raise_on_initializer_error=True) await get_scenario_run_service().reconcile_interrupted_runs_async() _, self.version = await self.source.read_with_version_async() + await self._restore_saved_entries_async(config) self._publish(config) except Exception: self.state, self.outcome = "restart-required", "initialization-failed" @@ -219,6 +254,7 @@ async def _apply_async(self, *, version: str) -> None: await config.apply_prepared_reinitialization_async(prepared=prepared) await self._management_async(config) self.version = current_version + await self._restore_saved_entries_async(config) self._publish(config) except (Exception, asyncio.CancelledError, BaseExceptionGroup) as error: logger.exception( diff --git a/pyrit/backend/services/scenario_configuration_resolver.py b/pyrit/backend/services/scenario_configuration_resolver.py index 42a78d86f6..fa5d941657 100644 --- a/pyrit/backend/services/scenario_configuration_resolver.py +++ b/pyrit/backend/services/scenario_configuration_resolver.py @@ -7,7 +7,15 @@ from typing import TYPE_CHECKING, Any -from pyrit.registry import ScenarioRegistry, ScorerRegistry, TargetRegistry +from pyrit.registry import ( + AttackTechniqueRegistry, + SavedEntryUnavailableError, + ScenarioRegistry, + ScorerRegistry, + TargetRegistry, + TechniqueSelectionUnavailableError, + UnavailableEntry, +) from pyrit.scenario.core import resolve_technique_tokens from pyrit.scenario.core.scenario_target_defaults import validate_default_adversarial_target from pyrit.score import TrueFalseScorer @@ -105,6 +113,41 @@ def resolve_target(*, target_name: str) -> PromptTarget: f"Target '{target_name}' not found in registry. Available targets: {', '.join(available_names)}" ) + @staticmethod + def _refuse_broad_selection_while_techniques_are_held( + *, + scenario_class: type[Scenario], + techniques: list[str] | None, + technique_override: TechniqueOverride | None, + ) -> None: + """ + Refuse choosing techniques by all, default, or a tag while a saved technique couldn't be restored. + + A broken technique's tags may be unknown, so such a selection could quietly leave it out. Naming + techniques still works, and scenarios whose techniques don't come from the registry aren't affected. + + Raises: + SavedEntryUnavailableError: If a named technique is held for a saved technique that couldn't be restored. + TechniqueSelectionUnavailableError: If the selection is broad while a saved technique is held. + """ + held = raise_if_a_named_technique_is_held(techniques) + if not held or "scenario_techniques" not in {p.name for p in scenario_class.supported_parameters()}: + return + selectors = [token.partition(":")[0] for token in techniques or []] + if not selectors and technique_override is not None: + return + try: + technique_class = scenario_class()._technique_class # type: ignore[ty:missing-argument] + except Exception: + technique_class = None + if technique_class is not None and getattr(technique_class, "_selection_catalog", None) is None: + return + aggregates = {"all", "default"} + if technique_class is not None: + aggregates |= {tag.casefold() for tag in technique_class.get_aggregate_tags()} + if not selectors or any(selector.casefold() in aggregates for selector in selectors): + raise TechniqueSelectionUnavailableError(names=[entry.name for entry in held if entry.name is not None]) + @classmethod def resolve_configuration( cls, @@ -131,6 +174,9 @@ def resolve_configuration( Raises: ValueError: If techniques or dataset overrides are invalid. """ + cls._refuse_broad_selection_while_techniques_are_held( + scenario_class=scenario_class, techniques=techniques, technique_override=technique_override + ) resolved: dict[str, Any] = {} if objective_target is not None: resolved["objective_target"] = objective_target @@ -213,3 +259,25 @@ def resolve_techniques_and_converters( if resolution.issues: raise ValueError(resolution.issues[0].message) return resolution.techniques, resolution.technique_converters + + +def raise_if_a_named_technique_is_held(techniques: list[str] | None) -> list[UnavailableEntry]: + """ + Refuse technique tokens that name a saved technique that couldn't be restored, in any letter case. + + Args: + techniques (list[str] | None): Technique tokens, each optionally followed by ``:`` and converters. + + Returns: + list[UnavailableEntry]: Every held saved technique, and a held folder of them. + + Raises: + SavedEntryUnavailableError: If a token names a held technique. + """ + held = AttackTechniqueRegistry.get_registry_singleton().instances.get_unavailable_entries() + held_by_name = {entry.name.casefold(): entry for entry in held if entry.name is not None} + for token in techniques or []: + name = token.partition(":")[0] + if (entry := held_by_name.get(name.casefold())) is not None: + raise SavedEntryUnavailableError(entry=entry, name=name) + return held diff --git a/pyrit/backend/services/scenario_run_service.py b/pyrit/backend/services/scenario_run_service.py index ed3b7d5aba..20c1a9016a 100644 --- a/pyrit/backend/services/scenario_run_service.py +++ b/pyrit/backend/services/scenario_run_service.py @@ -46,7 +46,10 @@ normalize_label_filters, ) from pyrit.backend.services.saved_conversation_read_service import SavedConversationReadService -from pyrit.backend.services.scenario_configuration_resolver import ScenarioConfigurationResolver +from pyrit.backend.services.scenario_configuration_resolver import ( + ScenarioConfigurationResolver, + raise_if_a_named_technique_is_held, +) from pyrit.backend.services.scenario_progress_read_model import ( ScenarioPlanLookup, ScenarioProgressReadModel, @@ -100,6 +103,7 @@ from pyrit.models.catalog.scenario import ScenarioOverloadSummary, ScenarioTargetSummary from pyrit.prompt_target import PromptTarget from pyrit.registry import InitializerRegistry, ScenarioRegistry +from pyrit.registry.instance_registry import SavedEntryReplacementError, record_saved_entry_refusals from pyrit.registry.resolution import reject_non_external_params, resolve_declared_params from pyrit.scenario import IncompatibleTechniquePolicy, Scenario from pyrit.scenario.core import get_default_adversarial_target, override_default_adversarial_target @@ -179,6 +183,10 @@ class ScenarioRunConflictError(ValueError): """A saved run cannot be admitted to the scheduler in its current state.""" +class SavedEntryLaunchConflictError(ScenarioRunConflictError): + """A launch script would replace an entry rebuilt from a saved recipe.""" + + class ScenarioRunNotFoundError(ValueError): """The requested saved run is not in the active memory database.""" @@ -443,6 +451,7 @@ def _selected_launch_factories( technique_class = metadata.technique_class if not isinstance(technique_class, type): return [local_factory] if local_factory is not None else [] + raise_if_a_named_technique_is_held(request.techniques) if configured_techniques is None: selectors = ( [technique_class(token.partition(":")[0]) for token in request.techniques] @@ -1614,6 +1623,7 @@ async def _run_initializers_async(self, *, request: RunScenarioRequest) -> None: Raises: ValueError: If an initializer name is not found in the registry. + SavedEntryLaunchConflictError: If an initializer would replace a saved entry. """ if not request.initializers: return @@ -1627,7 +1637,42 @@ async def _run_initializers_async(self, *, request: RunScenarioRequest) -> None: ) except KeyError as e: raise ValueError(f"Initializer not found: {e}") from None - await instance.initialize_async() + # The registries refuse to replace a saved entry; the record also catches a refusal + # that the script itself caught, which would leave the saved entry in its place. + with record_saved_entry_refusals() as refusals: + try: + await instance.initialize_async() + except SavedEntryReplacementError as error: + refusals.append((error.kind, error.name)) + except Exception as error: + # A script can wrap the refusal in an error of its own; the clash is still why it stopped. + if refusals: + raise self._saved_entry_conflict( + initializer_name=initializer_name, refusals=refusals + ) from error + raise + if refusals: + raise self._saved_entry_conflict(initializer_name=initializer_name, refusals=refusals) + + @staticmethod + def _saved_entry_conflict( + *, initializer_name: str, refusals: list[tuple[str, str]] + ) -> SavedEntryLaunchConflictError: + """ + Describe the saved entries a launch script would have replaced. + + Args: + initializer_name (str): The launch script's name. + refusals (list[tuple[str, str]]): The kind and name of each saved entry that was refused. + + Returns: + SavedEntryLaunchConflictError: The error to raise. + """ + entries = ", ".join(f"{kind.removesuffix('s')} '{name}'" for kind, name in dict.fromkeys(refusals)) + return SavedEntryLaunchConflictError( + f"The launch script '{initializer_name}' would replace the saved {entries}. " + "Leave that script out of the launch." + ) async def _initialize_scenario_async( self, diff --git a/pyrit/backend/services/scenario_suite_execution_service.py b/pyrit/backend/services/scenario_suite_execution_service.py index 1b85e89580..4691281ee6 100644 --- a/pyrit/backend/services/scenario_suite_execution_service.py +++ b/pyrit/backend/services/scenario_suite_execution_service.py @@ -13,7 +13,11 @@ from typing import TYPE_CHECKING from uuid import UUID, uuid4 -from pyrit.backend.services.scenario_run_service import ScenarioRunService, get_scenario_run_service +from pyrit.backend.services.scenario_run_service import ( + SavedEntryLaunchConflictError, + ScenarioRunService, + get_scenario_run_service, +) from pyrit.models import ( LaunchScenarioSuiteRequest, ScenarioRunState, @@ -25,9 +29,11 @@ ) from pyrit.registry import ( ResolvedScenarioSuiteEntry, + SavedEntryUnavailableError, ScenarioPresetRegistry, ScenarioSuiteLaunchResolver, ScenarioSuiteRegistry, + TechniqueSelectionUnavailableError, ) if TYPE_CHECKING: @@ -261,7 +267,7 @@ async def _submit_entries_async(self, *, state: _ExecutionState) -> None: ) async with state.lock: entry.state = ScenarioSuiteSubmissionState.SUBMISSION_FAILED - entry.error = self.SUBMISSION_FAILURE_MESSAGE + entry.error = self._describe_submission_failure(exc) entry.error_type = type(exc).__name__ state.updated_at = datetime.now(UTC) continue @@ -290,7 +296,7 @@ async def _submit_entries_async(self, *, state: _ExecutionState) -> None: except Exception as exc: async with state.lock: entry.state = ScenarioSuiteSubmissionState.SUBMISSION_FAILED - entry.error = self.SUBMISSION_FAILURE_MESSAGE + entry.error = self._describe_submission_failure(exc) entry.error_type = type(exc).__name__ state.updated_at = datetime.now(UTC) else: @@ -547,6 +553,23 @@ def _get_run_service(self) -> ScenarioRunService: """Return the injected run service or process singleton.""" return self._run_service if self._run_service is not None else get_scenario_run_service() + def _describe_submission_failure(self, error: Exception) -> str: + """ + Describe why an entry's run couldn't be created, without exposing internal error text. + + Args: + error (Exception): The error the run service raised. + + Returns: + str: The message to show for the entry. + """ + # These name only the launch script, entries and techniques, so they're safe to show. + if isinstance(error, (SavedEntryLaunchConflictError, TechniqueSelectionUnavailableError)): + return str(error) + if isinstance(error, SavedEntryUnavailableError): + return error.describe(with_details=False) + return self.SUBMISSION_FAILURE_MESSAGE + _service_instance: ScenarioSuiteExecutionService | None = None diff --git a/pyrit/backend/services/scenario_suite_service.py b/pyrit/backend/services/scenario_suite_service.py index 4e668bb4eb..2636d842d6 100644 --- a/pyrit/backend/services/scenario_suite_service.py +++ b/pyrit/backend/services/scenario_suite_service.py @@ -9,8 +9,6 @@ from functools import lru_cache from typing import Literal -from azure.core.exceptions import AzureError - from pyrit.backend.models import ( ScenarioSuiteEntryStatus, ScenarioSuiteListResponse, @@ -25,6 +23,7 @@ ScenarioSuiteRegistryEntry, ) from pyrit.registry import ScenarioPresetRegistry, ScenarioSuiteRegistry +from pyrit.registry.file_document_storage import DocumentStorageUnavailableError logger = logging.getLogger(__name__) @@ -209,7 +208,7 @@ def _resolve_preset_projection( ) stored_version = registry.get_preset_version(name=preset_name) - except (AzureError, OSError): + except DocumentStorageUnavailableError: logger.warning( "Could not project scenario preset '%s' because preset storage is unavailable.", preset_name, diff --git a/pyrit/common/azure_storage.py b/pyrit/common/azure_storage.py index d3ed6eeb4f..0195a24956 100644 --- a/pyrit/common/azure_storage.py +++ b/pyrit/common/azure_storage.py @@ -3,8 +3,14 @@ """Lightweight Azure Storage URI helpers.""" +import re from urllib.parse import parse_qs, urlparse +# A scheme followed by "://" and everything up to whitespace, so a quote can't end a URL before its query. +_URL_IN_TEXT = re.compile(r"\b[A-Za-z][A-Za-z0-9+.-]*://\S+") +# The signature of a SAS token, which can also appear in text without a scheme. +_SAS_SIGNATURE = re.compile(r"(?i)\bsig=[^&\s]+") + def is_azure_blob_uri(value: str, *, min_path_segments: int = 1) -> bool: """ @@ -48,9 +54,30 @@ def has_sas_signature(value: str) -> bool: def redact_url_credentials(value: str) -> str: """ - Remove query parameters and fragments from a URL for display or logging. + Remove the user name and password, query parameters, and fragment from a URL for display or logging. + + Returns: + str: The URL without them. + """ + parsed = urlparse(value) + return parsed._replace(netloc=parsed.netloc.rpartition("@")[2], query="", fragment="").geturl() + + +def redact_urls(text: str) -> str: + """ + Remove the user name and password, query parameters, and fragment from every URL in a text. + + A query or user name can hold a credential, such as a SAS token. A URL that can't be parsed is + hidden entirely, and a SAS signature is hidden wherever it appears. Returns: - str: The URL without query parameters or a fragment. + str: The text with each URL redacted. """ - return urlparse(value)._replace(query="", fragment="").geturl() + + def redact(match: re.Match[str]) -> str: + try: + return redact_url_credentials(match.group(0)) + except ValueError: + return "