FEAT: Rebuild saved registry entries in dependency order (part 3)

pyrit/backend/middleware/error_handlers.py

@@ -14,6 +14,7 @@
 from pyrit.backend.middleware.request_size import RequestTooLargeError, request_too_large_response
 from pyrit.backend.models import FieldError, ProblemDetail
 from pyrit.exceptions.analytics_exception import AnalyticsException
+from pyrit.registry.file_document_storage import DocumentStorageUnavailableError, DocumentUnreadableError
 
 logger = logging.getLogger(__name__)
 
@@ -169,6 +170,39 @@ async def permission_error_handler(  # pyrit-async-suffix-exempt
             content=problem.model_dump(exclude_none=True),
         )
 
+    @app.exception_handler(DocumentStorageUnavailableError)
+    async def storage_unavailable_handler(  # pyrit-async-suffix-exempt
+        request: Request,
+        exc: DocumentStorageUnavailableError,
+    ) -> JSONResponse:
+        """
+        Handle stored documents that can't be read or written as 503 Service Unavailable.
+
+        The log names the storage location, without credentials, and the reason, with the
+        traceback at debug level. The response never carries a path or URL.
+
+        Returns:
+            JSONResponse: RFC 7807 problem detail response with 503 status.
+        """
+        logger.warning(f"{exc} ({request.method} {request.url.path})")
+        logger.debug("Storage error details", exc_info=exc)
+        if isinstance(exc, DocumentUnreadableError):
+            detail = f"{exc.label} '{exc.name}' can't be read: {exc.reason}"
+        else:
+            detail = f"{exc.label} storage is unavailable"
+        problem = ProblemDetail(
+            type="/errors/storage-unavailable",
+            title="Service Unavailable",
+            status=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail=detail,
+            instance=str(request.url.path),
+        )
+
+        return JSONResponse(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            content=problem.model_dump(exclude_none=True),
+        )
+
     @app.exception_handler(NotImplementedError)
     async def not_implemented_handler(  # pyrit-async-suffix-exempt
         request: Request,

pyrit/backend/routes/initializers.py

@@ -14,7 +14,6 @@
     DELETE /api/initializers/{name}         — unregister an initializer
 """
 
-from azure.core.exceptions import AzureError
 from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
 
 from pyrit.backend.middleware.auth import require_admin
@@ -35,19 +34,6 @@
 router = APIRouter(prefix="/initializers", tags=["initializers"])
 
 
-def _custom_storage_unavailable() -> HTTPException:
-    """
-    Create a sanitized response for unavailable custom initializer storage.
-
-    Returns:
-        HTTPException: A service-unavailable response without SDK details.
-    """
-    return HTTPException(
-        status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-        detail="Custom initializer storage is temporarily unavailable",
-    )
-
-
 def _configured_initializers(request: Request) -> list[ConfiguredInitializerSetting]:
     """
     Read the initializer list captured from ``.pyrit_conf`` at backend startup.
@@ -138,10 +124,7 @@ async def list_custom_initializers(request: Request) -> CustomInitializerListRes
         CustomInitializerListResponse: The configured source and stored scripts.
     """
     _check_custom_initializers_allowed(request)
-    try:
-        return await get_initializer_service().list_custom_initializers_async()
-    except AzureError as exc:
-        raise _custom_storage_unavailable() from exc
+    return await get_initializer_service().list_custom_initializers_async()
 
 
 @router.get(
@@ -205,8 +188,6 @@ async def register_initializer(  # pyrit-async-suffix-exempt
 
     try:
         return await service.register_initializer_async(name=body.name, script_content=body.script_content)
-    except AzureError as exc:
-        raise _custom_storage_unavailable() from exc
     except ValueError as e:
         detail = str(e)
         if "already registered" in detail:
@@ -243,8 +224,6 @@ async def unregister_initializer(  # pyrit-async-suffix-exempt
 
     try:
         await service.unregister_initializer_async(initializer_name=initializer_name)
-    except AzureError as exc:
-        raise _custom_storage_unavailable() from exc
     except ValueError as e:
         raise HTTPException(
             status_code=status.HTTP_400_BAD_REQUEST,

pyrit/backend/routes/scenario_presets.py

@@ -19,7 +19,6 @@
 
 from typing import Annotated
 
-from azure.core.exceptions import AzureError
 from fastapi import APIRouter, Depends, HTTPException, Query, status
 
 from pyrit.backend.middleware.auth import current_user_name, require_admin
@@ -38,19 +37,6 @@
 router = APIRouter(prefix="/scenario-presets", tags=["scenario-presets"])
 
 
-def _storage_unavailable() -> HTTPException:
-    """
-    Create a sanitized response for unavailable preset storage.
-
-    Returns:
-        HTTPException: A service-unavailable response without SDK details.
-    """
-    return HTTPException(
-        status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-        detail="Scenario preset storage is temporarily unavailable",
-    )
-
-
 def _not_found(name: str) -> HTTPException:
     """
     Create a not-found response for a missing preset.
@@ -79,8 +65,6 @@ async def _load_preset_or_404_async(name: str) -> ScenarioPresetResponse:
     """
     try:
         preset = await get_scenario_preset_service().get_preset_async(name=name)
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
     except ValueError as exc:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None
 
@@ -106,10 +90,7 @@ async def list_scenario_presets(  # pyrit-async-suffix-exempt
     Returns:
         ScenarioPresetListResponse: The configured source and the stored presets.
     """
-    try:
-        return await get_scenario_preset_service().list_presets_async(include_estimates=include_estimates)
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
+    return await get_scenario_preset_service().list_presets_async(include_estimates=include_estimates)
 
 
 @router.get(
@@ -177,8 +158,6 @@ async def create_scenario_preset(  # pyrit-async-suffix-exempt
             status_code=status.HTTP_409_CONFLICT,
             detail=f"Scenario preset '{preset.name}' already exists",
         ) from exc
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
 
 
 @router.put(
@@ -239,8 +218,6 @@ async def update_scenario_preset(  # pyrit-async-suffix-exempt
             status_code=status.HTTP_409_CONFLICT,
             detail=f"Scenario preset '{name}' changed since it was read; re-read it and retry",
         ) from exc
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
 
 
 @router.delete(
@@ -261,8 +238,6 @@ async def delete_scenario_preset(name: IdentifierStr) -> None:  # pyrit-async-su
     """
     try:
         deleted = await get_scenario_preset_service().delete_preset_async(name=name)
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
     except ValueError as exc:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None
 
@@ -312,8 +287,6 @@ async def resolve_scenario_preset(  # pyrit-async-suffix-exempt
             status_code=status.HTTP_409_CONFLICT,
             detail=f"Scenario preset '{name}' changed since it was read; re-read it and retry",
         ) from None
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
     except ValueError as exc:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None
 

pyrit/backend/routes/scenario_suites.py

@@ -6,7 +6,6 @@
 from typing import Annotated
 from uuid import UUID
 
-from azure.core.exceptions import AzureError
 from fastapi import APIRouter, Depends, HTTPException, Request, status
 
 from pyrit.backend.middleware.auth import current_user_name, require_admin
@@ -31,19 +30,6 @@
 router = APIRouter(prefix="/scenario-suites", tags=["scenario-suites"])
 
 
-def _storage_unavailable() -> HTTPException:
-    """
-    Create a sanitized response for unavailable suite storage.
-
-    Returns:
-        HTTPException: A service-unavailable response without SDK details.
-    """
-    return HTTPException(
-        status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-        detail="Scenario suite storage is temporarily unavailable",
-    )
-
-
 def _not_found(name: str) -> HTTPException:
     """
     Create a not-found response for a missing suite.
@@ -66,8 +52,6 @@ async def _load_suite_or_404_async(name: str) -> ScenarioSuiteResponse:
     """
     try:
         suite = await get_scenario_suite_service().get_suite_async(name=name)
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
     except ValueError as exc:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None
 
@@ -84,10 +68,7 @@ async def list_scenario_suites() -> ScenarioSuiteListResponse:  # pyrit-async-su
     Returns:
         ScenarioSuiteListResponse: Suites from both origins with current entry statuses.
     """
-    try:
-        return await get_scenario_suite_service().list_suites_async()
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
+    return await get_scenario_suite_service().list_suites_async()
 
 
 @router.get(
@@ -181,8 +162,6 @@ async def launch_scenario_suite(  # pyrit-async-suffix-exempt
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None
     except ScenarioSuiteExecutionCapacityError as exc:
         raise HTTPException(status_code=status.HTTP_429_TOO_MANY_REQUESTS, detail=str(exc)) from None
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
     except RuntimeError as exc:
         raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc)) from None
 
@@ -239,8 +218,6 @@ async def create_scenario_suite(  # pyrit-async-suffix-exempt
             status_code=status.HTTP_409_CONFLICT,
             detail=f"Scenario suite '{suite.name}' already exists",
         ) from exc
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
 
 
 @router.put(
@@ -289,8 +266,6 @@ async def update_scenario_suite(  # pyrit-async-suffix-exempt
             status_code=status.HTTP_409_CONFLICT,
             detail=f"Scenario suite '{name}' changed since it was read; re-read it and retry",
         ) from exc
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
 
 
 @router.delete(
@@ -303,8 +278,6 @@ async def delete_scenario_suite(name: IdentifierStr) -> None:  # pyrit-async-suf
     """Delete one stored scenario suite."""
     try:
         deleted = await get_scenario_suite_service().delete_suite_async(name=name)
-    except AzureError as exc:
-        raise _storage_unavailable() from exc
     except ValueError as exc:
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None
 

pyrit/backend/services/scenario_suite_service.py

@@ -9,8 +9,6 @@
 from functools import lru_cache
 from typing import Literal
 
-from azure.core.exceptions import AzureError
-
 from pyrit.backend.models import (
     ScenarioSuiteEntryStatus,
     ScenarioSuiteListResponse,
@@ -25,6 +23,7 @@
     ScenarioSuiteRegistryEntry,
 )
 from pyrit.registry import ScenarioPresetRegistry, ScenarioSuiteRegistry
+from pyrit.registry.file_document_storage import DocumentStorageUnavailableError
 
 logger = logging.getLogger(__name__)
 
@@ -209,7 +208,7 @@ def _resolve_preset_projection(
                 )
 
             stored_version = registry.get_preset_version(name=preset_name)
-        except (AzureError, OSError):
+        except DocumentStorageUnavailableError:
             logger.warning(
                 "Could not project scenario preset '%s' because preset storage is unavailable.",
                 preset_name,

pyrit/common/azure_storage.py

@@ -3,8 +3,14 @@
 
 """Lightweight Azure Storage URI helpers."""
 
+import re
 from urllib.parse import parse_qs, urlparse
 
+# A scheme followed by "://" and everything up to whitespace, so a quote can't end a URL before its query.
+_URL_IN_TEXT = re.compile(r"\b[A-Za-z][A-Za-z0-9+.-]*://\S+")
+# The signature of a SAS token, which can also appear in text without a scheme.
+_SAS_SIGNATURE = re.compile(r"(?i)\bsig=[^&\s]+")
+
 
 def is_azure_blob_uri(value: str, *, min_path_segments: int = 1) -> bool:
     """
@@ -48,9 +54,30 @@ def has_sas_signature(value: str) -> bool:
 
 def redact_url_credentials(value: str) -> str:
     """
-    Remove query parameters and fragments from a URL for display or logging.
+    Remove the user name and password, query parameters, and fragment from a URL for display or logging.
+
+    Returns:
+        str: The URL without them.
+    """
+    parsed = urlparse(value)
+    return parsed._replace(netloc=parsed.netloc.rpartition("@")[2], query="", fragment="").geturl()
+
+
+def redact_urls(text: str) -> str:
+    """
+    Remove the user name and password, query parameters, and fragment from every URL in a text.
+
+    A query or user name can hold a credential, such as a SAS token. A URL that can't be parsed is
+    hidden entirely, and a SAS signature is hidden wherever it appears.
 
     Returns:
-        str: The URL without query parameters or a fragment.
+        str: The text with each URL redacted.
     """
-    return urlparse(value)._replace(query="", fragment="").geturl()
+
+    def redact(match: re.Match[str]) -> str:
+        try:
+            return redact_url_credentials(match.group(0))
+        except ValueError:
+            return "<URL hidden>"
+
+    return _SAS_SIGNATURE.sub("sig=<hidden>", _URL_IN_TEXT.sub(redact, text))

pyrit/models/catalog/__init__.py

@@ -20,6 +20,17 @@
 
 if TYPE_CHECKING:
     from pyrit.models.catalog.initializer import RegisteredInitializer
+    from pyrit.models.catalog.registry_recipe import (
+        ComponentRecipe,
+        IgnoredRecipeFile,
+        RegistryRecipeKind,
+        RegistryRestoreReport,
+        RestoreFailure,
+        RestoreFailureReason,
+        SavedEntryRef,
+        StoredRecipe,
+        TechniqueRecipe,
+    )
     from pyrit.models.catalog.scenario import (
         AttackErrorSummary,
         AttackRetrySummary,
@@ -60,9 +71,16 @@
 _LAZY_EXPORTS: dict[str, str] = {
     "AttackErrorSummary": "pyrit.models.catalog.scenario",
     "AttackRetrySummary": "pyrit.models.catalog.scenario",
+    "ComponentRecipe": "pyrit.models.catalog.registry_recipe",
+    "IgnoredRecipeFile": "pyrit.models.catalog.registry_recipe",
     "PresetEntry": "pyrit.models.catalog.scenario_preset",
     "PresetIssue": "pyrit.models.catalog.scenario_preset",
     "RegisteredInitializer": "pyrit.models.catalog.initializer",
+    "RegistryRecipeKind": "pyrit.models.catalog.registry_recipe",
+    "RegistryRestoreReport": "pyrit.models.catalog.registry_recipe",
+    "RestoreFailure": "pyrit.models.catalog.registry_recipe",
+    "RestoreFailureReason": "pyrit.models.catalog.registry_recipe",
+    "SavedEntryRef": "pyrit.models.catalog.registry_recipe",
     "ScorerInstance": "pyrit.models.catalog.scorer",
     "RegisteredScenario": "pyrit.models.catalog.scenario",
     "RunScenarioRequest": "pyrit.models.catalog.scenario",
@@ -87,8 +105,10 @@
     "ScenarioRunSummary": "pyrit.models.catalog.scenario",
     "ScenarioTechniqueSummary": "pyrit.models.catalog.scenario",
     "StoredPreset": "pyrit.models.catalog.scenario_preset",
+    "StoredRecipe": "pyrit.models.catalog.registry_recipe",
     "StoredScenarioSuite": "pyrit.models.catalog.scenario_suite",
     "TargetInstance": "pyrit.models.catalog.target",
+    "TechniqueRecipe": "pyrit.models.catalog.registry_recipe",
 }
 
 __all__ = list(_LAZY_EXPORTS)