diff --git a/.pyrit_conf_example b/.pyrit_conf_example index 51488ee0d8..f31a5feaec 100644 --- a/.pyrit_conf_example +++ b/.pyrit_conf_example @@ -161,6 +161,14 @@ allow_custom_initializers: false # as scenario_presets_source. Defaults to ~/.pyrit/scenario_suites. # scenario_suites_source: https://account.blob.core.windows.net/container/scenario_suites +# Optional storage for saved targets, converters, scorers, and techniques, so they +# come back when the backend restarts. Each kind has its own folder under the source. +# This supports the same local directory and Azure Blob sources as +# scenario_presets_source. If a saved entry can't be restored, the backend still +# starts, lists the failure in its runtime status, and refuses to use that entry's +# name until the file is fixed or deleted. Defaults to ~/.pyrit/registry_recipes. +# registry_recipes_source: https://account.blob.core.windows.net/container/registry_recipes + # Upload Folder for API-Created Targets # ------------------------------------- # Folder that targets created through the REST API may upload local files from diff --git a/doc/getting_started/pyrit_conf.md b/doc/getting_started/pyrit_conf.md index bba64b45d4..62a3827c43 100644 --- a/doc/getting_started/pyrit_conf.md +++ b/doc/getting_started/pyrit_conf.md @@ -194,6 +194,18 @@ Suites store ordered references to scenario presets and resolve each preset's cu A SAS query string may be included; otherwise, PyRIT uses `DefaultAzureCredential`. The default is `~/.pyrit/scenario_suites`. Local writes use the same hidden lock-file and atomic replacement behavior as scenario presets. Startup scans the configured source so malformed suite documents are reported immediately without preventing the backend from starting. +### `registry_recipes_source` + +Stores saved targets, converters, scorers, and techniques in a local directory or Azure Blob container, so they come back when the backend restarts. An Azure URI may include a blob-name prefix: + +```yaml +registry_recipes_source: https://account.blob.core.windows.net/pyrit-storage/registry-recipes +``` + +Each kind has its own folder under the source (`targets/`, `converters/`, `scorers/`, `techniques/`), with one JSON file per saved entry named after the entry. A file holds what's needed to rebuild the entry; restore refuses a file that stores a value for a parameter its class marks as a credential, such as `api_key`. A SAS query string may be included; otherwise, PyRIT uses `DefaultAzureCredential`. The default is `~/.pyrit/registry_recipes`. + +The backend restores saved entries at startup and after a live apply, once the configured initializers have registered theirs. If a saved entry can't be restored, the backend still becomes ready and lists each failure with its reason in `GET /api/config/runtime`. Every signed-in user can see the same list in `GET /api/restore-failures` and in the GUI's banner; users who aren't administrators get a short explanation of each reason instead of the full message. The failed entry keeps its name: looking it up, or starting a run that uses it, fails with the reason (409) instead of finding nothing or something else, and nothing else can take the name. While a saved technique is broken, a launch has to name its techniques, because choosing them by `all`, `default`, or a tag could quietly leave it out. If a whole folder can't be listed, nothing is restored and every name of that kind that isn't registered is held the same way. Fix or delete the listed files, then apply or restart. Files whose names aren't valid entry names (up to 64 lowercase letters, digits, and underscores, starting with a letter) are listed as ignored. + ### `initialization_scripts` Local paths to custom Python scripts containing `PyRITInitializer` subclasses. Paths can be absolute or relative to the current working directory. diff --git a/frontend/e2e/fixtures/manual_send_backend.py b/frontend/e2e/fixtures/manual_send_backend.py index 0c46ed2398..8be9b44941 100644 --- a/frontend/e2e/fixtures/manual_send_backend.py +++ b/frontend/e2e/fixtures/manual_send_backend.py @@ -4,6 +4,7 @@ """Offline backend for manual-send browser tests, with no real configuration or providers.""" import os +import tempfile from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from unittest.mock import AsyncMock, patch @@ -27,19 +28,22 @@ async def _initialize_offline_async(self: ConfigurationLoader, *, raise_on_initi @asynccontextmanager async def _offline_lifespan_async(application: FastAPI) -> AsyncGenerator[None, None]: - config = ConfigurationLoader(memory_db_type="in_memory", env_files=[], env_akv_ref=[]) - with ( - patch.dict(os.environ, {"PYRIT_AUTH_MODE": "local", "PYRIT_DEV_MODE": "true"}, clear=True), - patch.object( - ConfigurationFileService, - "_read_source_async", - new=AsyncMock(return_value="memory_db_type: in_memory\nenv_files: []\ninitializers: []\n"), - ), - patch.object(ConfigurationLoader, "load_with_overrides", return_value=config), - patch.object(ConfigurationLoader, "initialize_pyrit_async", new=_initialize_offline_async), - ): - async with lifespan(application): - yield + with tempfile.TemporaryDirectory() as recipes: + config = ConfigurationLoader( + memory_db_type="in_memory", env_files=[], env_akv_ref=[], registry_recipes_source=recipes + ) + with ( + patch.dict(os.environ, {"PYRIT_AUTH_MODE": "local", "PYRIT_DEV_MODE": "true"}, clear=True), + patch.object( + ConfigurationFileService, + "_read_source_async", + new=AsyncMock(return_value="memory_db_type: in_memory\nenv_files: []\ninitializers: []\n"), + ), + patch.object(ConfigurationLoader, "load_with_overrides", return_value=config), + patch.object(ConfigurationLoader, "initialize_pyrit_async", new=_initialize_offline_async), + ): + async with lifespan(application): + yield app.router.lifespan_context = _offline_lifespan_async diff --git a/frontend/src/hooks/useRuntime.styles.ts b/frontend/src/hooks/useRuntime.styles.ts new file mode 100644 index 0000000000..b0596a5a62 --- /dev/null +++ b/frontend/src/hooks/useRuntime.styles.ts @@ -0,0 +1,9 @@ +import { makeStyles } from '@fluentui/react-components' + +export const useRuntimeBannerStyles = makeStyles({ + // The app doesn't scroll, so a long list scrolls on its own. + failureList: { + maxHeight: '30vh', + overflowY: 'auto', + }, +}) diff --git a/frontend/src/hooks/useRuntime.test.tsx b/frontend/src/hooks/useRuntime.test.tsx index 9d592bbd89..91620a09ee 100644 --- a/frontend/src/hooks/useRuntime.test.tsx +++ b/frontend/src/hooks/useRuntime.test.tsx @@ -1,13 +1,22 @@ import { useState } from 'react' +import type { ReactNode } from 'react' -import { act, render, screen, waitFor } from '@testing-library/react' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { act, render, screen, waitFor, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { runtimeApi } from '@/services/api' +import type { RestoreFailure } from '@/types' -import { RuntimeProvider, useRuntime } from './useRuntime' +import { RuntimeBanner, RuntimeProvider, useRuntime } from './useRuntime' -jest.mock('@/services/api', () => ({ runtimeApi: { getReadiness: jest.fn() } })) +jest.mock('@/services/api', () => ({ + runtimeApi: { getReadiness: jest.fn(), getRestoreFailures: jest.fn() }, +})) + +function TestWrapper({ children }: { children: ReactNode }) { + return {children} +} function Consumer() { const runtime = useRuntime() @@ -41,3 +50,173 @@ describe('RuntimeProvider', () => { expect(screen.getByRole('textbox', { name: 'Draft' })).toHaveValue('keep my draft') }) }) + +function brokenTarget(name: string): RestoreFailure { + return { kind: 'targets', name, reason: 'unknown_class', message: 'It is broken.', version: 'v1', dependency: null } +} + +function renderBanner() { + render() +} + +describe('RuntimeBanner', () => { + beforeEach(() => { jest.clearAllMocks() }) + + it('lists the saved entries that could not be restored while the runtime is ready', async () => { + jest.mocked(runtimeApi.getReadiness).mockResolvedValue({ + ready: true, + state: 'ready', + generation: 'first', + restore_failures: 2, + }) + jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({ + items: [ + { + kind: 'targets', + name: 'pool', + reason: 'missing_dependency', + message: 'It uses an entry that does not exist.', + version: 'v1', + dependency: { kind: 'targets', name: 'gone' }, + }, + { + kind: 'converters', + name: null, + reason: 'unreadable_storage', + message: 'The folder could not be read.', + version: null, + dependency: null, + }, + ], + }) + + renderBanner() + + const list = await screen.findByRole('list', { name: "Saved entries that couldn't be restored" }) + expect(await within(list).findByText('Target pool: It uses an entry that does not exist. (target gone)')).toBeInTheDocument() + expect(within(list).getByText('Saved converters: The folder could not be read.')).toBeInTheDocument() + expect(screen.getByText(/fixes or\s+deletes their saved files, then applies the configuration/)).toBeInTheDocument() + expect(screen.queryByText(/has to name its techniques/)).not.toBeInTheDocument() + expect(screen.queryByRole('button')).not.toBeInTheDocument() + }) + + it('says a scan has to name its techniques while a saved technique is broken', async () => { + jest.mocked(runtimeApi.getReadiness).mockResolvedValue({ + ready: true, + state: 'ready', + generation: 'first', + restore_failures: 1, + }) + jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({ + items: [{ ...brokenTarget('formal'), kind: 'techniques' }], + }) + + renderBanner() + + expect(await screen.findByText(/a scan has to name its techniques/)).toBeInTheDocument() + }) + + it('shows nothing and asks for no list when every saved entry was restored', async () => { + jest.mocked(runtimeApi.getReadiness).mockResolvedValue({ + ready: true, + state: 'ready', + generation: 'first', + restore_failures: 0, + }) + + renderBanner() + + await waitFor(() => expect(runtimeApi.getReadiness).toHaveBeenCalled()) + expect(screen.queryByRole('list')).not.toBeInTheDocument() + expect(runtimeApi.getRestoreFailures).not.toHaveBeenCalled() + }) + + it('does not ask for restore failures while the runtime is not ready', async () => { + jest.mocked(runtimeApi.getReadiness).mockResolvedValue({ + ready: false, + state: 'restart-required', + generation: '', + restore_failures: 1, + }) + + renderBanner() + + expect(await screen.findByText(/PyRIT runtime: restart-required/)).toBeInTheDocument() + expect(screen.queryByRole('list')).not.toBeInTheDocument() + expect(runtimeApi.getRestoreFailures).not.toHaveBeenCalled() + }) + + it('reads the list again only when the count or the generation changes', async () => { + jest.useFakeTimers() + const getReadiness = jest.mocked(runtimeApi.getReadiness) + getReadiness.mockResolvedValue({ ready: true, state: 'ready', generation: 'first', restore_failures: 1 }) + jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({ items: [brokenTarget('ghost')] }) + + renderBanner() + expect(await screen.findByText('Target ghost: It is broken.')).toBeInTheDocument() + await act(async () => { jest.advanceTimersByTime(4_000) }) + expect(runtimeApi.getRestoreFailures).toHaveBeenCalledTimes(1) + + jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({ items: [brokenTarget('other')] }) + getReadiness.mockResolvedValue({ ready: true, state: 'ready', generation: 'second', restore_failures: 1 }) + await act(async () => { jest.advanceTimersByTime(2_000) }) + + expect(await screen.findByText('Target other: It is broken.')).toBeInTheDocument() + expect(runtimeApi.getRestoreFailures).toHaveBeenCalledTimes(2) + jest.useRealTimers() + }) + + it('hides the list once nothing fails any more', async () => { + jest.useFakeTimers() + const getReadiness = jest.mocked(runtimeApi.getReadiness) + getReadiness.mockResolvedValue({ ready: true, state: 'ready', generation: 'first', restore_failures: 1 }) + jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({ items: [brokenTarget('ghost')] }) + + renderBanner() + expect(await screen.findByText('Target ghost: It is broken.')).toBeInTheDocument() + getReadiness.mockResolvedValue({ ready: true, state: 'ready', generation: 'second', restore_failures: 0 }) + await act(async () => { jest.advanceTimersByTime(2_000) }) + + await waitFor(() => expect(screen.queryByText('Target ghost: It is broken.')).not.toBeInTheDocument()) + jest.useRealTimers() + }) + + it('tries again when the list could not be loaded, keeping the last list', async () => { + jest.useFakeTimers() + jest.mocked(runtimeApi.getReadiness).mockResolvedValue({ + ready: true, + state: 'ready', + generation: 'first', + restore_failures: 1, + }) + jest.mocked(runtimeApi.getRestoreFailures) + .mockRejectedValueOnce(new Error('Network Error')) + .mockResolvedValue({ items: [brokenTarget('ghost')] }) + + renderBanner() + expect(await screen.findByText(/list of saved entries couldn't be loaded/)).toBeInTheDocument() + await act(async () => { jest.advanceTimersByTime(2_000) }) + + expect(await screen.findByText('Target ghost: It is broken.')).toBeInTheDocument() + expect(screen.queryByText(/list of saved entries couldn't be loaded/)).not.toBeInTheDocument() + jest.useRealTimers() + }) + + it('lets a long list scroll on its own', async () => { + jest.mocked(runtimeApi.getReadiness).mockResolvedValue({ + ready: true, + state: 'ready', + generation: 'first', + restore_failures: 40, + }) + jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({ + items: Array.from({ length: 40 }, (_, index) => brokenTarget(`target_${index}`)), + }) + + renderBanner() + + const list = await screen.findByRole('list', { name: "Saved entries that couldn't be restored" }) + expect(list).toHaveAttribute('tabindex', '0') + expect(within(list).getAllByRole('listitem')).toHaveLength(40) + }) +}) diff --git a/frontend/src/hooks/useRuntime.tsx b/frontend/src/hooks/useRuntime.tsx index 2035358d63..95966600b7 100644 --- a/frontend/src/hooks/useRuntime.tsx +++ b/frontend/src/hooks/useRuntime.tsx @@ -4,9 +4,17 @@ import type { ReactNode } from 'react' import { MessageBar, MessageBarBody } from '@fluentui/react-components' import { runtimeApi } from '@/services/api' -import type { RuntimeReadiness } from '@/types' +import type { RegistryRecipeKind, RestoreFailure, RuntimeReadiness } from '@/types' + +import { useRuntimeBannerStyles } from './useRuntime.styles' const POLL_INTERVAL_MS = 2_000 +const KIND_LABELS: Record = { + targets: 'Target', + converters: 'Converter', + scorers: 'Scorer', + techniques: 'Technique', +} const RuntimeContext = createContext({ ready: true, state: 'ready', generation: '' }) // eslint-disable-next-line react-refresh/only-export-components @@ -38,9 +46,84 @@ export function RuntimeProvider({ children }: { children: ReactNode }) { return {children} } +interface RestoreFailures { + failures: RestoreFailure[] + listFailed: boolean +} + +// The list is read when the count or the runtime generation changes, rather than on every readiness poll. +function useRestoreFailures(active: boolean, count: number, generation: string): RestoreFailures { + const [failures, setFailures] = useState([]) + const [listFailed, setListFailed] = useState(false) + useEffect(() => { + if (!active) return + let cancelled = false + let retryTimer: ReturnType | undefined + const load = async (): Promise => { + try { + const list = await runtimeApi.getRestoreFailures() + if (!cancelled) { + setFailures(list.items) + setListFailed(false) + } + } catch { + // Keep the last list and say it couldn't be refreshed; try again shortly. + if (!cancelled) { + setListFailed(true) + retryTimer = setTimeout(() => { void load() }, POLL_INTERVAL_MS) + } + } + } + void load() + return () => { + cancelled = true + clearTimeout(retryTimer) + } + }, [active, count, generation]) + return { failures: active ? failures : [], listFailed: active && listFailed } +} + +function describeFailure(failure: RestoreFailure): string { + const subject = failure.name === null ? `Saved ${failure.kind}` : `${KIND_LABELS[failure.kind]} ${failure.name}` + const dependency = failure.dependency + ? ` (${KIND_LABELS[failure.dependency.kind].toLowerCase()} ${failure.dependency.name})` + : '' + return `${subject}: ${failure.message}${dependency}` +} + export function RuntimeBanner() { const runtime = useRuntime() - if (runtime.ready) return null + const failureCount = runtime.restore_failures ?? 0 + const styles = useRuntimeBannerStyles() + const { failures, listFailed } = useRestoreFailures( + runtime.ready && failureCount > 0, + failureCount, + runtime.generation, + ) + if (runtime.ready) { + if (failureCount === 0) return null + const techniqueFailed = failures.some((failure) => failure.kind === 'techniques') + return ( + + + Some saved entries couldn't be restored, so they can't be used until an administrator fixes or + deletes their saved files, then applies the configuration or restarts the backend. + {techniqueFailed && ' Until then, a scan has to name its techniques instead of choosing all, default, or a tag.'} +
+ {failures.length > 0 && ( + // Focusable so the list can be scrolled from the keyboard. +
    + {failures.map((failure) => ( +
  • {describeFailure(failure)}
    • + ))} +
+ )} + {listFailed &&
The list of saved entries couldn't be loaded. Trying again.
} +
+ + + ) + } return ( diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 10d01b2b9a..116f8808dc 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -377,6 +377,9 @@ export const runtimeApi = { getReadiness: async (): Promise => { return (await apiClient.get('/runtime')).data }, + getRestoreFailures: async (): Promise => { + return (await apiClient.get('/restore-failures')).data + }, } export const targetsApi = { @@ -994,4 +997,4 @@ export const datasetsApi = { }, } -import type { RuntimeReadiness, RuntimeStatus } from '@/types' +import type { RestoreFailureList, RuntimeReadiness, RuntimeStatus } from '@/types' diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 59054dab85..cb2ebb5585 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -2391,6 +2391,30 @@ export interface RuntimeReadiness { ready: boolean state: string generation: string + /** How many saved entries couldn't be restored; the list has the details. */ + restore_failures?: number +} + +export type RegistryRecipeKind = 'targets' | 'converters' | 'scorers' | 'techniques' + +export interface SavedEntryRef { + kind: RegistryRecipeKind + name: string +} + +/** A saved entry, or a whole folder of them when `name` is null, that couldn't be restored. */ +export interface RestoreFailure { + kind: RegistryRecipeKind + name: string | null + reason: string + /** Full details for administrators; a fixed explanation of the reason for everyone else. */ + message: string + version: string | null + dependency: SavedEntryRef | null +} + +export interface RestoreFailureList { + items: RestoreFailure[] } export interface RuntimeStatus { diff --git a/pyrit/backend/README.md b/pyrit/backend/README.md index d0b8ea33ab..47a9427ab8 100644 --- a/pyrit/backend/README.md +++ b/pyrit/backend/README.md @@ -337,8 +337,9 @@ The chat preserves failed drafts across replacement, but requires converter choi to be reviewed again rather than restoring outputs tied to the previous registry. The offline browser fixture `frontend.e2e.fixtures.manual_send_backend:app` uses the real -backend lifecycle with isolated in-memory SQLite, no environment files, and no default -providers. The recovery tests register deterministic loopback targets. For example, start +backend lifecycle with isolated in-memory SQLite, no environment files, no saved registry +entries, and no default providers. The recovery tests register deterministic loopback +targets. For example, start it with `uv run python -m uvicorn frontend.e2e.fixtures.manual_send_backend:app --host 127.0.0.1 --port 18213` (set `PYTHONUTF8=1` on Windows). In another shell, set `PYRIT_BACKEND_URL=http://127.0.0.1:18213` and `E2E_FRONTEND_PORT=31213`, then run `npx playwright test chat-recovery --project seeded --workers 1` diff --git a/pyrit/backend/main.py b/pyrit/backend/main.py index 15aa89814a..f4f1914e4a 100644 --- a/pyrit/backend/main.py +++ b/pyrit/backend/main.py @@ -40,6 +40,7 @@ media, message_sends, operations, + restore_failures, scenario_presets, scenario_suites, scenarios, @@ -151,6 +152,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: app.include_router(initializers.router, prefix="/api", tags=["initializers"]) app.include_router(labels.router, prefix="/api", tags=["labels"]) app.include_router(health.router, prefix="/api", tags=["health"]) +app.include_router(restore_failures.router, prefix="/api", tags=["restore-failures"]) app.include_router(auth.router, prefix="/api", tags=["auth"]) app.include_router(media.router, prefix="/api", tags=["media"]) app.include_router(scores.router, prefix="/api", tags=["scores"]) diff --git a/pyrit/backend/middleware/error_handlers.py b/pyrit/backend/middleware/error_handlers.py index 5973d6289e..9186f41531 100644 --- a/pyrit/backend/middleware/error_handlers.py +++ b/pyrit/backend/middleware/error_handlers.py @@ -11,9 +11,12 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse +from pyrit.backend.middleware.auth import has_admin_access from pyrit.backend.middleware.request_size import RequestTooLargeError, request_too_large_response from pyrit.backend.models import FieldError, ProblemDetail from pyrit.exceptions.analytics_exception import AnalyticsException +from pyrit.registry import SavedEntryUnavailableError, TechniqueSelectionUnavailableError +from pyrit.registry.file_document_storage import DocumentStorageUnavailableError, DocumentUnreadableError logger = logging.getLogger(__name__) @@ -169,6 +172,68 @@ async def permission_error_handler( # pyrit-async-suffix-exempt content=problem.model_dump(exclude_none=True), ) + @app.exception_handler(DocumentStorageUnavailableError) + async def storage_unavailable_handler( # pyrit-async-suffix-exempt + request: Request, + exc: DocumentStorageUnavailableError, + ) -> JSONResponse: + """ + Handle stored documents that can't be read or written as 503 Service Unavailable. + + The log names the storage location, without credentials, and the reason, with the + traceback at debug level. The response never carries a path or URL. + + Returns: + JSONResponse: RFC 7807 problem detail response with 503 status. + """ + logger.warning(f"{exc} ({request.method} {request.url.path})") + logger.debug("Storage error details", exc_info=exc) + if isinstance(exc, DocumentUnreadableError): + detail = f"{exc.label} '{exc.name}' can't be read: {exc.reason}" + else: + detail = f"{exc.label} storage is unavailable" + problem = ProblemDetail( + type="/errors/storage-unavailable", + title="Service Unavailable", + status=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=detail, + instance=str(request.url.path), + ) + + return JSONResponse( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + content=problem.model_dump(exclude_none=True), + ) + + @app.exception_handler(SavedEntryUnavailableError) + async def saved_entry_unavailable_handler( # pyrit-async-suffix-exempt + request: Request, + exc: SavedEntryUnavailableError, + ) -> JSONResponse: + """ + Handle a name held for a saved entry that couldn't be restored as 409 Conflict. + + Administrators get the full message; anyone else gets a fixed explanation of the reason, + since the message can quote a server setting. + + Returns: + JSONResponse: RFC 7807 problem detail response with 409 status. + """ + return _conflict(request=request, detail=exc.describe(with_details=_shows_details(request))) + + @app.exception_handler(TechniqueSelectionUnavailableError) + async def technique_selection_unavailable_handler( # pyrit-async-suffix-exempt + request: Request, + exc: TechniqueSelectionUnavailableError, + ) -> JSONResponse: + """ + Handle a broad technique selection made while a saved technique couldn't be restored as 409 Conflict. + + Returns: + JSONResponse: RFC 7807 problem detail response with 409 status. + """ + return _conflict(request=request, detail=str(exc)) + @app.exception_handler(NotImplementedError) async def not_implemented_handler( # pyrit-async-suffix-exempt request: Request, @@ -223,3 +288,33 @@ async def generic_exception_handler( # pyrit-async-suffix-exempt status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content=problem.model_dump(exclude_none=True), ) + + +def _conflict(*, request: Request, detail: str) -> JSONResponse: + """ + Build a 409 Conflict problem response. + + Returns: + JSONResponse: RFC 7807 problem detail response with 409 status. + """ + problem = ProblemDetail( + type="/errors/conflict", + title="Conflict", + status=status.HTTP_409_CONFLICT, + detail=detail, + instance=str(request.url.path), + ) + return JSONResponse(status_code=status.HTTP_409_CONFLICT, content=problem.model_dump(exclude_none=True)) + + +def _shows_details(request: Request) -> bool: + """ + Return whether the caller may see a restore failure's full message, which can quote a server setting. + + Returns: + bool: True for an administrator; False otherwise, including when no authentication policy is set. + """ + try: + return has_admin_access(request) + except RuntimeError: + return False diff --git a/pyrit/backend/routes/attacks.py b/pyrit/backend/routes/attacks.py index 51cc1d4dc0..76bac6db9f 100644 --- a/pyrit/backend/routes/attacks.py +++ b/pyrit/backend/routes/attacks.py @@ -45,6 +45,7 @@ from pyrit.backend.services.media_persistence import MediaAccessDeniedError from pyrit.common import print_deprecation_message from pyrit.memory.memory_interface import AttackStateConflictError +from pyrit.registry import SavedEntryUnavailableError logger = logging.getLogger(__name__) @@ -666,6 +667,8 @@ async def add_message( # pyrit-async-suffix-exempt status_code=status.HTTP_400_BAD_REQUEST, detail=error_msg, ) from e + except SavedEntryUnavailableError: + raise except Exception as e: logger.exception("Failed to add message to attack '%s'", attack_result_id) raise HTTPException( diff --git a/pyrit/backend/routes/converters.py b/pyrit/backend/routes/converters.py index 10d129394e..d97841170e 100644 --- a/pyrit/backend/routes/converters.py +++ b/pyrit/backend/routes/converters.py @@ -24,6 +24,7 @@ UnregisteredConverter, ) from pyrit.backend.services import get_converter_service +from pyrit.registry import SavedEntryUnavailableError router = APIRouter(prefix="/converters", tags=["converters"]) @@ -89,6 +90,8 @@ async def create_converter( status_code=status.HTTP_400_BAD_REQUEST, detail=str(e), ) from e + except SavedEntryUnavailableError: + raise except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -165,6 +168,8 @@ async def preview_conversion(request: ConverterPreviewRequest) -> ConverterPrevi status_code=status.HTTP_400_BAD_REQUEST, detail=str(e), ) from e + except SavedEntryUnavailableError: + raise except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/pyrit/backend/routes/health.py b/pyrit/backend/routes/health.py index b6e06c91a4..e0cd361faa 100644 --- a/pyrit/backend/routes/health.py +++ b/pyrit/backend/routes/health.py @@ -13,30 +13,31 @@ router = APIRouter() -def _runtime_readiness(request: Request) -> dict[str, str | bool]: +def _runtime_readiness(request: Request) -> dict[str, str | bool | int]: runtime = getattr(request.app.state, "runtime_lifecycle", None) if runtime is None: - return {"ready": False, "state": "failed", "generation": ""} + return {"ready": False, "state": "failed", "generation": "", "restore_failures": 0} return { "ready": runtime.reported_state == "ready", "state": runtime.reported_state, "generation": runtime.generation, + "restore_failures": len(runtime.restore_failures), } @router.get("/runtime") -async def runtime_readiness_async(request: Request) -> dict[str, str | bool]: +async def runtime_readiness_async(request: Request) -> dict[str, str | bool | int]: """ - Expose readiness and generation without administrative details. + Expose readiness, generation, and how many saved entries couldn't be restored, without details. Returns: - dict[str, str | bool]: Lightweight runtime readiness. + dict[str, str | bool | int]: Lightweight runtime readiness. """ return _runtime_readiness(request) @router.get("/health") -async def health_check_async(*, request: Request, response: Response) -> dict[str, str | bool | None]: +async def health_check_async(*, request: Request, response: Response) -> dict[str, str | bool | int | None]: """ Check the health status of the backend service. @@ -48,7 +49,7 @@ async def health_check_async(*, request: Request, response: Response) -> dict[st deployment checks use the readiness fields instead. Returns: - dict[str, str | bool | None]: Health status, timestamp, runtime readiness, and serving ACA revision. + dict[str, str | bool | int | None]: Health status, timestamp, runtime readiness, and serving ACA revision. """ readiness = _runtime_readiness(request) response.headers["Cache-Control"] = "no-store" diff --git a/pyrit/backend/routes/initializers.py b/pyrit/backend/routes/initializers.py index 50fa79db85..529f92d641 100644 --- a/pyrit/backend/routes/initializers.py +++ b/pyrit/backend/routes/initializers.py @@ -14,7 +14,6 @@ DELETE /api/initializers/{name} — unregister an initializer """ -from azure.core.exceptions import AzureError from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from pyrit.backend.middleware.auth import require_admin @@ -35,19 +34,6 @@ router = APIRouter(prefix="/initializers", tags=["initializers"]) -def _custom_storage_unavailable() -> HTTPException: - """ - Create a sanitized response for unavailable custom initializer storage. - - Returns: - HTTPException: A service-unavailable response without SDK details. - """ - return HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Custom initializer storage is temporarily unavailable", - ) - - def _configured_initializers(request: Request) -> list[ConfiguredInitializerSetting]: """ Read the initializer list captured from ``.pyrit_conf`` at backend startup. @@ -138,10 +124,7 @@ async def list_custom_initializers(request: Request) -> CustomInitializerListRes CustomInitializerListResponse: The configured source and stored scripts. """ _check_custom_initializers_allowed(request) - try: - return await get_initializer_service().list_custom_initializers_async() - except AzureError as exc: - raise _custom_storage_unavailable() from exc + return await get_initializer_service().list_custom_initializers_async() @router.get( @@ -205,8 +188,6 @@ async def register_initializer( # pyrit-async-suffix-exempt try: return await service.register_initializer_async(name=body.name, script_content=body.script_content) - except AzureError as exc: - raise _custom_storage_unavailable() from exc except ValueError as e: detail = str(e) if "already registered" in detail: @@ -243,8 +224,6 @@ async def unregister_initializer( # pyrit-async-suffix-exempt try: await service.unregister_initializer_async(initializer_name=initializer_name) - except AzureError as exc: - raise _custom_storage_unavailable() from exc except ValueError as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, diff --git a/pyrit/backend/routes/message_sends.py b/pyrit/backend/routes/message_sends.py index 297783c99e..7441f28af5 100644 --- a/pyrit/backend/routes/message_sends.py +++ b/pyrit/backend/routes/message_sends.py @@ -15,6 +15,7 @@ from pyrit.backend.services.manual_send_scheduler import ManualSendConflictError, ManualSendQueueFullError from pyrit.backend.services.message_send_service import MessageSendNotFoundError, get_message_send_service from pyrit.memory.memory_interface import AttackStateConflictError +from pyrit.registry import SavedEntryUnavailableError logger = logging.getLogger(__name__) router = APIRouter(prefix="/attacks", tags=["attacks"]) @@ -49,6 +50,8 @@ async def submit_message_send_async( except ValueError as exc: is_not_found = "not found" in str(exc).lower() or "does not exist" in str(exc).lower() raise HTTPException(status_code=404 if is_not_found else 400, detail=str(exc)) from exc + except SavedEntryUnavailableError: + raise except Exception as exc: logger.exception("Failed to accept message send for attack '%s'", attack_result_id) raise HTTPException(status_code=500, detail="Could not accept the message send. Check server logs.") from exc diff --git a/pyrit/backend/routes/restore_failures.py b/pyrit/backend/routes/restore_failures.py new file mode 100644 index 0000000000..a9f25a1805 --- /dev/null +++ b/pyrit/backend/routes/restore_failures.py @@ -0,0 +1,38 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Saved registry entries that couldn't be restored, listed for every signed-in user.""" + +from typing import TYPE_CHECKING, Any + +from fastapi import APIRouter, Request + +from pyrit.backend.middleware.auth import has_admin_access + +if TYPE_CHECKING: + from pyrit.models.catalog import RestoreFailure + +router = APIRouter(prefix="/restore-failures", tags=["restore-failures"]) + + +@router.get("") +async def list_restore_failures_async(request: Request) -> dict[str, list[dict[str, Any]]]: + """ + List the saved entries that couldn't be restored and why. + + Administrators get the full message. Other users get a fixed explanation of the reason instead, + since a constructor's error can quote a server setting. + + Returns: + dict[str, list[dict[str, Any]]]: The failures, under ``items``. + """ + runtime = getattr(request.app.state, "runtime_lifecycle", None) + failures: list[RestoreFailure] = runtime.restore_failures if runtime is not None else [] + show_details = has_admin_access(request) + items = [] + for failure in failures: + item = failure.model_dump(mode="json") + if not show_details: + item["message"] = failure.reason.explanation + items.append(item) + return {"items": items} diff --git a/pyrit/backend/routes/scenario_presets.py b/pyrit/backend/routes/scenario_presets.py index 4b3fc1db00..973112df7b 100644 --- a/pyrit/backend/routes/scenario_presets.py +++ b/pyrit/backend/routes/scenario_presets.py @@ -19,7 +19,6 @@ from typing import Annotated -from azure.core.exceptions import AzureError from fastapi import APIRouter, Depends, HTTPException, Query, status from pyrit.backend.middleware.auth import current_user_name, require_admin @@ -38,19 +37,6 @@ router = APIRouter(prefix="/scenario-presets", tags=["scenario-presets"]) -def _storage_unavailable() -> HTTPException: - """ - Create a sanitized response for unavailable preset storage. - - Returns: - HTTPException: A service-unavailable response without SDK details. - """ - return HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Scenario preset storage is temporarily unavailable", - ) - - def _not_found(name: str) -> HTTPException: """ Create a not-found response for a missing preset. @@ -79,8 +65,6 @@ async def _load_preset_or_404_async(name: str) -> ScenarioPresetResponse: """ try: preset = await get_scenario_preset_service().get_preset_async(name=name) - except AzureError as exc: - raise _storage_unavailable() from exc except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None @@ -106,10 +90,7 @@ async def list_scenario_presets( # pyrit-async-suffix-exempt Returns: ScenarioPresetListResponse: The configured source and the stored presets. """ - try: - return await get_scenario_preset_service().list_presets_async(include_estimates=include_estimates) - except AzureError as exc: - raise _storage_unavailable() from exc + return await get_scenario_preset_service().list_presets_async(include_estimates=include_estimates) @router.get( @@ -177,8 +158,6 @@ async def create_scenario_preset( # pyrit-async-suffix-exempt status_code=status.HTTP_409_CONFLICT, detail=f"Scenario preset '{preset.name}' already exists", ) from exc - except AzureError as exc: - raise _storage_unavailable() from exc @router.put( @@ -239,8 +218,6 @@ async def update_scenario_preset( # pyrit-async-suffix-exempt status_code=status.HTTP_409_CONFLICT, detail=f"Scenario preset '{name}' changed since it was read; re-read it and retry", ) from exc - except AzureError as exc: - raise _storage_unavailable() from exc @router.delete( @@ -261,8 +238,6 @@ async def delete_scenario_preset(name: IdentifierStr) -> None: # pyrit-async-su """ try: deleted = await get_scenario_preset_service().delete_preset_async(name=name) - except AzureError as exc: - raise _storage_unavailable() from exc except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None @@ -312,8 +287,6 @@ async def resolve_scenario_preset( # pyrit-async-suffix-exempt status_code=status.HTTP_409_CONFLICT, detail=f"Scenario preset '{name}' changed since it was read; re-read it and retry", ) from None - except AzureError as exc: - raise _storage_unavailable() from exc except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None diff --git a/pyrit/backend/routes/scenario_suites.py b/pyrit/backend/routes/scenario_suites.py index 0b4fa7e09a..f1ec16872e 100644 --- a/pyrit/backend/routes/scenario_suites.py +++ b/pyrit/backend/routes/scenario_suites.py @@ -6,7 +6,6 @@ from typing import Annotated from uuid import UUID -from azure.core.exceptions import AzureError from fastapi import APIRouter, Depends, HTTPException, Request, status from pyrit.backend.middleware.auth import current_user_name, require_admin @@ -31,19 +30,6 @@ router = APIRouter(prefix="/scenario-suites", tags=["scenario-suites"]) -def _storage_unavailable() -> HTTPException: - """ - Create a sanitized response for unavailable suite storage. - - Returns: - HTTPException: A service-unavailable response without SDK details. - """ - return HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Scenario suite storage is temporarily unavailable", - ) - - def _not_found(name: str) -> HTTPException: """ Create a not-found response for a missing suite. @@ -66,8 +52,6 @@ async def _load_suite_or_404_async(name: str) -> ScenarioSuiteResponse: """ try: suite = await get_scenario_suite_service().get_suite_async(name=name) - except AzureError as exc: - raise _storage_unavailable() from exc except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None @@ -84,10 +68,7 @@ async def list_scenario_suites() -> ScenarioSuiteListResponse: # pyrit-async-su Returns: ScenarioSuiteListResponse: Suites from both origins with current entry statuses. """ - try: - return await get_scenario_suite_service().list_suites_async() - except AzureError as exc: - raise _storage_unavailable() from exc + return await get_scenario_suite_service().list_suites_async() @router.get( @@ -181,8 +162,6 @@ async def launch_scenario_suite( # pyrit-async-suffix-exempt raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None except ScenarioSuiteExecutionCapacityError as exc: raise HTTPException(status_code=status.HTTP_429_TOO_MANY_REQUESTS, detail=str(exc)) from None - except AzureError as exc: - raise _storage_unavailable() from exc except RuntimeError as exc: raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc)) from None @@ -239,8 +218,6 @@ async def create_scenario_suite( # pyrit-async-suffix-exempt status_code=status.HTTP_409_CONFLICT, detail=f"Scenario suite '{suite.name}' already exists", ) from exc - except AzureError as exc: - raise _storage_unavailable() from exc @router.put( @@ -289,8 +266,6 @@ async def update_scenario_suite( # pyrit-async-suffix-exempt status_code=status.HTTP_409_CONFLICT, detail=f"Scenario suite '{name}' changed since it was read; re-read it and retry", ) from exc - except AzureError as exc: - raise _storage_unavailable() from exc @router.delete( @@ -303,8 +278,6 @@ async def delete_scenario_suite(name: IdentifierStr) -> None: # pyrit-async-suf """Delete one stored scenario suite.""" try: deleted = await get_scenario_suite_service().delete_suite_async(name=name) - except AzureError as exc: - raise _storage_unavailable() from exc except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None diff --git a/pyrit/backend/routes/scorers.py b/pyrit/backend/routes/scorers.py index 445f2830b3..65287aea6e 100644 --- a/pyrit/backend/routes/scorers.py +++ b/pyrit/backend/routes/scorers.py @@ -17,6 +17,7 @@ ) from pyrit.backend.services.scorer_service import get_scorer_service from pyrit.models.catalog import ScorerInstance +from pyrit.registry import SavedEntryUnavailableError router = APIRouter(prefix="/scorers", tags=["scorers"]) @@ -65,6 +66,8 @@ async def create_scorer( return await get_scorer_service().create_scorer_async(request=request) except ValueError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + except SavedEntryUnavailableError: + raise except Exception as exc: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/pyrit/backend/routes/targets.py b/pyrit/backend/routes/targets.py index c520a42e05..a10164b4bc 100644 --- a/pyrit/backend/routes/targets.py +++ b/pyrit/backend/routes/targets.py @@ -23,6 +23,7 @@ ) from pyrit.backend.services import get_target_service from pyrit.models.catalog import TargetInstance +from pyrit.registry import SavedEntryUnavailableError router = APIRouter(prefix="/targets", tags=["targets"]) @@ -132,6 +133,8 @@ async def create_target( status_code=status.HTTP_400_BAD_REQUEST, detail=str(e), ) from e + except SavedEntryUnavailableError: + raise except Exception as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/pyrit/backend/services/runtime_lifecycle.py b/pyrit/backend/services/runtime_lifecycle.py index daef21f0ed..c6ae1be279 100644 --- a/pyrit/backend/services/runtime_lifecycle.py +++ b/pyrit/backend/services/runtime_lifecycle.py @@ -7,7 +7,7 @@ import logging import os import uuid -from typing import Any +from typing import TYPE_CHECKING, Any from fastapi import FastAPI @@ -26,10 +26,14 @@ from pyrit.common.path import CONFIGURATION_DIRECTORY_PATH from pyrit.memory import CentralMemory from pyrit.registry import InitializerRegistry, ScenarioPresetRegistry, ScenarioSuiteRegistry +from pyrit.registry.registry_recipe_restore import RegistryRecipeRestorer from pyrit.setup import ConfigurationLoader from pyrit.setup.environment_loading import resolve_environment_async from pyrit.setup.initialization import validate_reinitialization_memory +if TYPE_CHECKING: + from pyrit.models.catalog import IgnoredRecipeFile, RestoreFailure + logger = logging.getLogger(__name__) @@ -46,6 +50,8 @@ def __init__(self, *, app: FastAPI, source: ConfigurationFileService) -> None: self.version: str | None = None self.outcome = "starting" self.message = "" + self.restore_failures: list[RestoreFailure] = [] + self.restore_ignored: list[IgnoredRecipeFile] = [] self.operations: set[asyncio.Task[None]] = set() self.management_operations: set[asyncio.Task[None]] = set() self.apply_task: asyncio.Task[None] | None = None @@ -75,6 +81,8 @@ def status(self) -> dict[str, Any]: "message": self.message, "enabled": self.topology_supported, "applying": self.apply_task is not None and not self.apply_task.done(), + "restore_failures": [failure.model_dump(mode="json") for failure in self.restore_failures], + "restore_ignored": [file.model_dump(mode="json") for file in self.restore_ignored], } async def _load_async(self) -> ConfigurationLoader: @@ -119,6 +127,32 @@ def _publish(self, config: ConfigurationLoader) -> None: self.generation = str(uuid.uuid4()) self.state, self.outcome, self.message = "ready", "success", "PyRIT is ready." + async def _restore_saved_entries_async(self, config: ConfigurationLoader) -> None: + """ + Rebuild saved registry entries after the startup scripts have registered theirs. + + The runtime becomes ready even when some can't be restored. Each one that failed keeps its name, + so looking it up or starting a run that uses it fails with the reason instead of finding nothing + or something else, and the failures are listed in the runtime status. + + Args: + config (ConfigurationLoader): The loaded configuration naming the storage source. + """ + restorer = RegistryRecipeRestorer(source=config.registry_recipes_source) + # Reading can wait on storage, so it runs in a worker thread. Entries are built on this loop, as + # the target API builds them, because a constructor such as HuggingFaceChatTarget's starts a task + # on it. A slow constructor holds the loop while it runs. + saved = await asyncio.to_thread(restorer.read) + report = restorer.restore(saved=saved) + self.restore_failures, self.restore_ignored = report.failures, report.ignored + if report.failures: + count = len(report.failures) + logger.warning( + "%d saved registry %s couldn't be restored. Fix or delete them; see GET /api/config/runtime.", + count, + "entry" if count == 1 else "entries", + ) + async def startup_async(self) -> None: """Keep management alive even when parsing or initialization fails.""" self.app.state.allow_custom_initializers = False @@ -137,6 +171,7 @@ async def startup_async(self) -> None: await config.initialize_pyrit_async(raise_on_initializer_error=True) await get_scenario_run_service().reconcile_interrupted_runs_async() _, self.version = await self.source.read_with_version_async() + await self._restore_saved_entries_async(config) self._publish(config) except Exception: self.state, self.outcome = "restart-required", "initialization-failed" @@ -219,6 +254,7 @@ async def _apply_async(self, *, version: str) -> None: await config.apply_prepared_reinitialization_async(prepared=prepared) await self._management_async(config) self.version = current_version + await self._restore_saved_entries_async(config) self._publish(config) except (Exception, asyncio.CancelledError, BaseExceptionGroup) as error: logger.exception( diff --git a/pyrit/backend/services/scenario_configuration_resolver.py b/pyrit/backend/services/scenario_configuration_resolver.py index 42a78d86f6..fa5d941657 100644 --- a/pyrit/backend/services/scenario_configuration_resolver.py +++ b/pyrit/backend/services/scenario_configuration_resolver.py @@ -7,7 +7,15 @@ from typing import TYPE_CHECKING, Any -from pyrit.registry import ScenarioRegistry, ScorerRegistry, TargetRegistry +from pyrit.registry import ( + AttackTechniqueRegistry, + SavedEntryUnavailableError, + ScenarioRegistry, + ScorerRegistry, + TargetRegistry, + TechniqueSelectionUnavailableError, + UnavailableEntry, +) from pyrit.scenario.core import resolve_technique_tokens from pyrit.scenario.core.scenario_target_defaults import validate_default_adversarial_target from pyrit.score import TrueFalseScorer @@ -105,6 +113,41 @@ def resolve_target(*, target_name: str) -> PromptTarget: f"Target '{target_name}' not found in registry. Available targets: {', '.join(available_names)}" ) + @staticmethod + def _refuse_broad_selection_while_techniques_are_held( + *, + scenario_class: type[Scenario], + techniques: list[str] | None, + technique_override: TechniqueOverride | None, + ) -> None: + """ + Refuse choosing techniques by all, default, or a tag while a saved technique couldn't be restored. + + A broken technique's tags may be unknown, so such a selection could quietly leave it out. Naming + techniques still works, and scenarios whose techniques don't come from the registry aren't affected. + + Raises: + SavedEntryUnavailableError: If a named technique is held for a saved technique that couldn't be restored. + TechniqueSelectionUnavailableError: If the selection is broad while a saved technique is held. + """ + held = raise_if_a_named_technique_is_held(techniques) + if not held or "scenario_techniques" not in {p.name for p in scenario_class.supported_parameters()}: + return + selectors = [token.partition(":")[0] for token in techniques or []] + if not selectors and technique_override is not None: + return + try: + technique_class = scenario_class()._technique_class # type: ignore[ty:missing-argument] + except Exception: + technique_class = None + if technique_class is not None and getattr(technique_class, "_selection_catalog", None) is None: + return + aggregates = {"all", "default"} + if technique_class is not None: + aggregates |= {tag.casefold() for tag in technique_class.get_aggregate_tags()} + if not selectors or any(selector.casefold() in aggregates for selector in selectors): + raise TechniqueSelectionUnavailableError(names=[entry.name for entry in held if entry.name is not None]) + @classmethod def resolve_configuration( cls, @@ -131,6 +174,9 @@ def resolve_configuration( Raises: ValueError: If techniques or dataset overrides are invalid. """ + cls._refuse_broad_selection_while_techniques_are_held( + scenario_class=scenario_class, techniques=techniques, technique_override=technique_override + ) resolved: dict[str, Any] = {} if objective_target is not None: resolved["objective_target"] = objective_target @@ -213,3 +259,25 @@ def resolve_techniques_and_converters( if resolution.issues: raise ValueError(resolution.issues[0].message) return resolution.techniques, resolution.technique_converters + + +def raise_if_a_named_technique_is_held(techniques: list[str] | None) -> list[UnavailableEntry]: + """ + Refuse technique tokens that name a saved technique that couldn't be restored, in any letter case. + + Args: + techniques (list[str] | None): Technique tokens, each optionally followed by ``:`` and converters. + + Returns: + list[UnavailableEntry]: Every held saved technique, and a held folder of them. + + Raises: + SavedEntryUnavailableError: If a token names a held technique. + """ + held = AttackTechniqueRegistry.get_registry_singleton().instances.get_unavailable_entries() + held_by_name = {entry.name.casefold(): entry for entry in held if entry.name is not None} + for token in techniques or []: + name = token.partition(":")[0] + if (entry := held_by_name.get(name.casefold())) is not None: + raise SavedEntryUnavailableError(entry=entry, name=name) + return held diff --git a/pyrit/backend/services/scenario_run_service.py b/pyrit/backend/services/scenario_run_service.py index ed3b7d5aba..466e1fc397 100644 --- a/pyrit/backend/services/scenario_run_service.py +++ b/pyrit/backend/services/scenario_run_service.py @@ -46,7 +46,10 @@ normalize_label_filters, ) from pyrit.backend.services.saved_conversation_read_service import SavedConversationReadService -from pyrit.backend.services.scenario_configuration_resolver import ScenarioConfigurationResolver +from pyrit.backend.services.scenario_configuration_resolver import ( + ScenarioConfigurationResolver, + raise_if_a_named_technique_is_held, +) from pyrit.backend.services.scenario_progress_read_model import ( ScenarioPlanLookup, ScenarioProgressReadModel, @@ -443,6 +446,7 @@ def _selected_launch_factories( technique_class = metadata.technique_class if not isinstance(technique_class, type): return [local_factory] if local_factory is not None else [] + raise_if_a_named_technique_is_held(request.techniques) if configured_techniques is None: selectors = ( [technique_class(token.partition(":")[0]) for token in request.techniques] diff --git a/pyrit/backend/services/scenario_suite_service.py b/pyrit/backend/services/scenario_suite_service.py index 4e668bb4eb..2636d842d6 100644 --- a/pyrit/backend/services/scenario_suite_service.py +++ b/pyrit/backend/services/scenario_suite_service.py @@ -9,8 +9,6 @@ from functools import lru_cache from typing import Literal -from azure.core.exceptions import AzureError - from pyrit.backend.models import ( ScenarioSuiteEntryStatus, ScenarioSuiteListResponse, @@ -25,6 +23,7 @@ ScenarioSuiteRegistryEntry, ) from pyrit.registry import ScenarioPresetRegistry, ScenarioSuiteRegistry +from pyrit.registry.file_document_storage import DocumentStorageUnavailableError logger = logging.getLogger(__name__) @@ -209,7 +208,7 @@ def _resolve_preset_projection( ) stored_version = registry.get_preset_version(name=preset_name) - except (AzureError, OSError): + except DocumentStorageUnavailableError: logger.warning( "Could not project scenario preset '%s' because preset storage is unavailable.", preset_name, diff --git a/pyrit/common/azure_storage.py b/pyrit/common/azure_storage.py index d3ed6eeb4f..0195a24956 100644 --- a/pyrit/common/azure_storage.py +++ b/pyrit/common/azure_storage.py @@ -3,8 +3,14 @@ """Lightweight Azure Storage URI helpers.""" +import re from urllib.parse import parse_qs, urlparse +# A scheme followed by "://" and everything up to whitespace, so a quote can't end a URL before its query. +_URL_IN_TEXT = re.compile(r"\b[A-Za-z][A-Za-z0-9+.-]*://\S+") +# The signature of a SAS token, which can also appear in text without a scheme. +_SAS_SIGNATURE = re.compile(r"(?i)\bsig=[^&\s]+") + def is_azure_blob_uri(value: str, *, min_path_segments: int = 1) -> bool: """ @@ -48,9 +54,30 @@ def has_sas_signature(value: str) -> bool: def redact_url_credentials(value: str) -> str: """ - Remove query parameters and fragments from a URL for display or logging. + Remove the user name and password, query parameters, and fragment from a URL for display or logging. + + Returns: + str: The URL without them. + """ + parsed = urlparse(value) + return parsed._replace(netloc=parsed.netloc.rpartition("@")[2], query="", fragment="").geturl() + + +def redact_urls(text: str) -> str: + """ + Remove the user name and password, query parameters, and fragment from every URL in a text. + + A query or user name can hold a credential, such as a SAS token. A URL that can't be parsed is + hidden entirely, and a SAS signature is hidden wherever it appears. Returns: - str: The URL without query parameters or a fragment. + str: The text with each URL redacted. """ - return urlparse(value)._replace(query="", fragment="").geturl() + + def redact(match: re.Match[str]) -> str: + try: + return redact_url_credentials(match.group(0)) + except ValueError: + return "