FEAT: Recover from a failed restore of saved registry entries without a restart (part 5)

.pyrit_conf_example

@@ -161,6 +161,14 @@ allow_custom_initializers: false
 # as scenario_presets_source. Defaults to ~/.pyrit/scenario_suites.
 # scenario_suites_source: https://account.blob.core.windows.net/container/scenario_suites
 
+# Optional storage for saved targets, converters, scorers, and techniques, so they
+# come back when the backend restarts. Each kind has its own folder under the source.
+# This supports the same local directory and Azure Blob sources as
+# scenario_presets_source. If a saved entry can't be restored, the backend still
+# starts, lists the failure in its runtime status, and refuses to use that entry's
+# name until the file is fixed or deleted. Defaults to ~/.pyrit/registry_recipes.
+# registry_recipes_source: https://account.blob.core.windows.net/container/registry_recipes
+
 # Upload Folder for API-Created Targets
 # -------------------------------------
 # Folder that targets created through the REST API may upload local files from

doc/getting_started/pyrit_conf.md

@@ -194,6 +194,18 @@ Suites store ordered references to scenario presets and resolve each preset's cu
 
 A SAS query string may be included; otherwise, PyRIT uses `DefaultAzureCredential`. The default is `~/.pyrit/scenario_suites`. Local writes use the same hidden lock-file and atomic replacement behavior as scenario presets. Startup scans the configured source so malformed suite documents are reported immediately without preventing the backend from starting.
 
+### `registry_recipes_source`
+
+Stores saved targets, converters, scorers, and techniques in a local directory or Azure Blob container, so they come back when the backend restarts. An Azure URI may include a blob-name prefix:
+
+```yaml
+registry_recipes_source: https://account.blob.core.windows.net/pyrit-storage/registry-recipes
+```
+
+Each kind has its own folder under the source (`targets/`, `converters/`, `scorers/`, `techniques/`), with one JSON file per saved entry named after the entry. A file holds what's needed to rebuild the entry; restore refuses a file that stores a value for a parameter its class marks as a credential, such as `api_key`. A SAS query string may be included; otherwise, PyRIT uses `DefaultAzureCredential`. The default is `~/.pyrit/registry_recipes`.
+
+The backend restores saved entries at startup and after a live apply, once the configured initializers have registered theirs. If a saved entry can't be restored, the backend still becomes ready and lists each failure with its reason in `GET /api/config/runtime`. Every signed-in user can see the same list in `GET /api/restore-failures` and in the GUI's banner; users who aren't administrators get a short explanation of each reason instead of the full message. The failed entry keeps its name: looking it up, or starting a run that uses it, fails with the reason (409) instead of finding nothing or something else, and nothing else can take the name. While a saved technique is broken, a launch has to name its techniques, because choosing them by `all`, `default`, or a tag could quietly leave it out. If a whole folder can't be listed, nothing is restored and every name of that kind that isn't registered is held the same way. Fix or delete the listed files, then apply or restart. Files whose names aren't valid entry names (up to 64 lowercase letters, digits, and underscores, starting with a letter) are listed as ignored.
+
 ### `initialization_scripts`
 
 Local paths to custom Python scripts containing `PyRITInitializer` subclasses. Paths can be absolute or relative to the current working directory.

frontend/e2e/fixtures/manual_send_backend.py

@@ -4,6 +4,7 @@
 """Offline backend for manual-send browser tests, with no real configuration or providers."""
 
 import os
+import tempfile
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from unittest.mock import AsyncMock, patch
@@ -27,19 +28,22 @@ async def _initialize_offline_async(self: ConfigurationLoader, *, raise_on_initi
 
 @asynccontextmanager
 async def _offline_lifespan_async(application: FastAPI) -> AsyncGenerator[None, None]:
-    config = ConfigurationLoader(memory_db_type="in_memory", env_files=[], env_akv_ref=[])
-    with (
-        patch.dict(os.environ, {"PYRIT_AUTH_MODE": "local", "PYRIT_DEV_MODE": "true"}, clear=True),
-        patch.object(
-            ConfigurationFileService,
-            "_read_source_async",
-            new=AsyncMock(return_value="memory_db_type: in_memory\nenv_files: []\ninitializers: []\n"),
-        ),
-        patch.object(ConfigurationLoader, "load_with_overrides", return_value=config),
-        patch.object(ConfigurationLoader, "initialize_pyrit_async", new=_initialize_offline_async),
-    ):
-        async with lifespan(application):
-            yield
+    with tempfile.TemporaryDirectory() as recipes:
+        config = ConfigurationLoader(
+            memory_db_type="in_memory", env_files=[], env_akv_ref=[], registry_recipes_source=recipes
+        )
+        with (
+            patch.dict(os.environ, {"PYRIT_AUTH_MODE": "local", "PYRIT_DEV_MODE": "true"}, clear=True),
+            patch.object(
+                ConfigurationFileService,
+                "_read_source_async",
+                new=AsyncMock(return_value="memory_db_type: in_memory\nenv_files: []\ninitializers: []\n"),
+            ),
+            patch.object(ConfigurationLoader, "load_with_overrides", return_value=config),
+            patch.object(ConfigurationLoader, "initialize_pyrit_async", new=_initialize_offline_async),
+        ):
+            async with lifespan(application):
+                yield
 
 
 app.router.lifespan_context = _offline_lifespan_async

frontend/src/hooks/useRuntime.styles.ts

@@ -0,0 +1,9 @@
+import { makeStyles } from '@fluentui/react-components'
+
+export const useRuntimeBannerStyles = makeStyles({
+  // The app doesn't scroll, so a long list scrolls on its own.
+  failureList: {
+    maxHeight: '30vh',
+    overflowY: 'auto',
+  },
+})

frontend/src/hooks/useRuntime.test.tsx

@@ -1,13 +1,22 @@
 import { useState } from 'react'
+import type { ReactNode } from 'react'
 
-import { act, render, screen, waitFor } from '@testing-library/react'
+import { FluentProvider, webLightTheme } from '@fluentui/react-components'
+import { act, render, screen, waitFor, within } from '@testing-library/react'
 import userEvent from '@testing-library/user-event'
 
 import { runtimeApi } from '@/services/api'
+import type { RestoreFailure } from '@/types'
 
-import { RuntimeProvider, useRuntime } from './useRuntime'
+import { RuntimeBanner, RuntimeProvider, useRuntime } from './useRuntime'
 
-jest.mock('@/services/api', () => ({ runtimeApi: { getReadiness: jest.fn() } }))
+jest.mock('@/services/api', () => ({
+  runtimeApi: { getReadiness: jest.fn(), getRestoreFailures: jest.fn() },
+}))
+
+function TestWrapper({ children }: { children: ReactNode }) {
+  return <FluentProvider theme={webLightTheme}>{children}</FluentProvider>
+}
 
 function Consumer() {
   const runtime = useRuntime()
@@ -41,3 +50,173 @@ describe('RuntimeProvider', () => {
     expect(screen.getByRole('textbox', { name: 'Draft' })).toHaveValue('keep my draft')
   })
 })
+
+function brokenTarget(name: string): RestoreFailure {
+  return { kind: 'targets', name, reason: 'unknown_class', message: 'It is broken.', version: 'v1', dependency: null }
+}
+
+function renderBanner() {
+  render(<TestWrapper><RuntimeProvider><RuntimeBanner /></RuntimeProvider></TestWrapper>)
+}
+
+describe('RuntimeBanner', () => {
+  beforeEach(() => { jest.clearAllMocks() })
+
+  it('lists the saved entries that could not be restored while the runtime is ready', async () => {
+    jest.mocked(runtimeApi.getReadiness).mockResolvedValue({
+      ready: true,
+      state: 'ready',
+      generation: 'first',
+      restore_failures: 2,
+    })
+    jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({
+      items: [
+        {
+          kind: 'targets',
+          name: 'pool',
+          reason: 'missing_dependency',
+          message: 'It uses an entry that does not exist.',
+          version: 'v1',
+          dependency: { kind: 'targets', name: 'gone' },
+        },
+        {
+          kind: 'converters',
+          name: null,
+          reason: 'unreadable_storage',
+          message: 'The folder could not be read.',
+          version: null,
+          dependency: null,
+        },
+      ],
+    })
+
+    renderBanner()
+
+    const list = await screen.findByRole('list', { name: "Saved entries that couldn't be restored" })
+    expect(await within(list).findByText('Target pool: It uses an entry that does not exist. (target gone)')).toBeInTheDocument()
+    expect(within(list).getByText('Saved converters: The folder could not be read.')).toBeInTheDocument()
+    expect(screen.getByText(/fixes or\s+deletes their saved files, then applies the configuration/)).toBeInTheDocument()
+    expect(screen.queryByText(/has to name its techniques/)).not.toBeInTheDocument()
+    expect(screen.queryByRole('button')).not.toBeInTheDocument()
+  })
+
+  it('says a scan has to name its techniques while a saved technique is broken', async () => {
+    jest.mocked(runtimeApi.getReadiness).mockResolvedValue({
+      ready: true,
+      state: 'ready',
+      generation: 'first',
+      restore_failures: 1,
+    })
+    jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({
+      items: [{ ...brokenTarget('formal'), kind: 'techniques' }],
+    })
+
+    renderBanner()
+
+    expect(await screen.findByText(/a scan has to name its techniques/)).toBeInTheDocument()
+  })
+
+  it('shows nothing and asks for no list when every saved entry was restored', async () => {
+    jest.mocked(runtimeApi.getReadiness).mockResolvedValue({
+      ready: true,
+      state: 'ready',
+      generation: 'first',
+      restore_failures: 0,
+    })
+
+    renderBanner()
+
+    await waitFor(() => expect(runtimeApi.getReadiness).toHaveBeenCalled())
+    expect(screen.queryByRole('list')).not.toBeInTheDocument()
+    expect(runtimeApi.getRestoreFailures).not.toHaveBeenCalled()
+  })
+
+  it('does not ask for restore failures while the runtime is not ready', async () => {
+    jest.mocked(runtimeApi.getReadiness).mockResolvedValue({
+      ready: false,
+      state: 'restart-required',
+      generation: '',
+      restore_failures: 1,
+    })
+
+    renderBanner()
+
+    expect(await screen.findByText(/PyRIT runtime: restart-required/)).toBeInTheDocument()
+    expect(screen.queryByRole('list')).not.toBeInTheDocument()
+    expect(runtimeApi.getRestoreFailures).not.toHaveBeenCalled()
+  })
+
+  it('reads the list again only when the count or the generation changes', async () => {
+    jest.useFakeTimers()
+    const getReadiness = jest.mocked(runtimeApi.getReadiness)
+    getReadiness.mockResolvedValue({ ready: true, state: 'ready', generation: 'first', restore_failures: 1 })
+    jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({ items: [brokenTarget('ghost')] })
+
+    renderBanner()
+    expect(await screen.findByText('Target ghost: It is broken.')).toBeInTheDocument()
+    await act(async () => { jest.advanceTimersByTime(4_000) })
+    expect(runtimeApi.getRestoreFailures).toHaveBeenCalledTimes(1)
+
+    jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({ items: [brokenTarget('other')] })
+    getReadiness.mockResolvedValue({ ready: true, state: 'ready', generation: 'second', restore_failures: 1 })
+    await act(async () => { jest.advanceTimersByTime(2_000) })
+
+    expect(await screen.findByText('Target other: It is broken.')).toBeInTheDocument()
+    expect(runtimeApi.getRestoreFailures).toHaveBeenCalledTimes(2)
+    jest.useRealTimers()
+  })
+
+  it('hides the list once nothing fails any more', async () => {
+    jest.useFakeTimers()
+    const getReadiness = jest.mocked(runtimeApi.getReadiness)
+    getReadiness.mockResolvedValue({ ready: true, state: 'ready', generation: 'first', restore_failures: 1 })
+    jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({ items: [brokenTarget('ghost')] })
+
+    renderBanner()
+    expect(await screen.findByText('Target ghost: It is broken.')).toBeInTheDocument()
+    getReadiness.mockResolvedValue({ ready: true, state: 'ready', generation: 'second', restore_failures: 0 })
+    await act(async () => { jest.advanceTimersByTime(2_000) })
+
+    await waitFor(() => expect(screen.queryByText('Target ghost: It is broken.')).not.toBeInTheDocument())
+    jest.useRealTimers()
+  })
+
+  it('tries again when the list could not be loaded, keeping the last list', async () => {
+    jest.useFakeTimers()
+    jest.mocked(runtimeApi.getReadiness).mockResolvedValue({
+      ready: true,
+      state: 'ready',
+      generation: 'first',
+      restore_failures: 1,
+    })
+    jest.mocked(runtimeApi.getRestoreFailures)
+      .mockRejectedValueOnce(new Error('Network Error'))
+      .mockResolvedValue({ items: [brokenTarget('ghost')] })
+
+    renderBanner()
+    expect(await screen.findByText(/list of saved entries couldn't be loaded/)).toBeInTheDocument()
+    await act(async () => { jest.advanceTimersByTime(2_000) })
+
+    expect(await screen.findByText('Target ghost: It is broken.')).toBeInTheDocument()
+    expect(screen.queryByText(/list of saved entries couldn't be loaded/)).not.toBeInTheDocument()
+    jest.useRealTimers()
+  })
+
+  it('lets a long list scroll on its own', async () => {
+    jest.mocked(runtimeApi.getReadiness).mockResolvedValue({
+      ready: true,
+      state: 'ready',
+      generation: 'first',
+      restore_failures: 40,
+    })
+    jest.mocked(runtimeApi.getRestoreFailures).mockResolvedValue({
+      items: Array.from({ length: 40 }, (_, index) => brokenTarget(`target_${index}`)),
+    })
+
+    renderBanner()
+
+    const list = await screen.findByRole('list', { name: "Saved entries that couldn't be restored" })
+    expect(list).toHaveAttribute('tabindex', '0')
+    expect(within(list).getAllByRole('listitem')).toHaveLength(40)
+  })
+})

frontend/src/hooks/useRuntime.tsx

@@ -4,9 +4,17 @@
 import { MessageBar, MessageBarBody } from '@fluentui/react-components'
 
 import { runtimeApi } from '@/services/api'
-import type { RuntimeReadiness } from '@/types'
+import type { RegistryRecipeKind, RestoreFailure, RuntimeReadiness } from '@/types'
+
+import { useRuntimeBannerStyles } from './useRuntime.styles'
 
 const POLL_INTERVAL_MS = 2_000
+const KIND_LABELS: Record<RegistryRecipeKind, string> = {
+  targets: 'Target',
+  converters: 'Converter',
+  scorers: 'Scorer',
+  techniques: 'Technique',
+}
 const RuntimeContext = createContext<RuntimeReadiness>({ ready: true, state: 'ready', generation: '' })
 
 // eslint-disable-next-line react-refresh/only-export-components
@@ -38,9 +46,84 @@
   return <RuntimeContext.Provider value={runtime}>{children}</RuntimeContext.Provider>
 }
 
+interface RestoreFailures {
+  failures: RestoreFailure[]
+  listFailed: boolean
+}
+
+// The list is read when the count or the runtime generation changes, rather than on every readiness poll.
+function useRestoreFailures(active: boolean, count: number, generation: string): RestoreFailures {
+  const [failures, setFailures] = useState<RestoreFailure[]>([])
+  const [listFailed, setListFailed] = useState(false)
+  useEffect(() => {
+    if (!active) return
+    let cancelled = false
+    let retryTimer: ReturnType<typeof setTimeout> | undefined
+    const load = async (): Promise<void> => {
+      try {
+        const list = await runtimeApi.getRestoreFailures()
+        if (!cancelled) {
+          setFailures(list.items)
+          setListFailed(false)
+        }
+      } catch {
+        // Keep the last list and say it couldn't be refreshed; try again shortly.
+        if (!cancelled) {
+          setListFailed(true)
+          retryTimer = setTimeout(() => { void load() }, POLL_INTERVAL_MS)
+        }
+      }
+    }
+    void load()
+    return () => {
+      cancelled = true
+      clearTimeout(retryTimer)
+    }
+  }, [active, count, generation])
+  return { failures: active ? failures : [], listFailed: active && listFailed }
+}
+
+function describeFailure(failure: RestoreFailure): string {
+  const subject = failure.name === null ? `Saved ${failure.kind}` : `${KIND_LABELS[failure.kind]} ${failure.name}`
+  const dependency = failure.dependency
+    ? ` (${KIND_LABELS[failure.dependency.kind].toLowerCase()} ${failure.dependency.name})`
+    : ''
+  return `${subject}: ${failure.message}${dependency}`
+}
+
 export function RuntimeBanner() {
   const runtime = useRuntime()
-  if (runtime.ready) return null
+  const failureCount = runtime.restore_failures ?? 0
+  const styles = useRuntimeBannerStyles()
+  const { failures, listFailed } = useRestoreFailures(
+    runtime.ready && failureCount > 0,
+    failureCount,
+    runtime.generation,
+  )
+  if (runtime.ready) {
+    if (failureCount === 0) return null
+    const techniqueFailed = failures.some((failure) => failure.kind === 'techniques')
+    return (
+      <MessageBar intent="warning">
+        <MessageBarBody>
+          Some saved entries couldn&apos;t be restored, so they can&apos;t be used until an administrator fixes or
+          deletes their saved files, then applies the configuration or restarts the backend.
+          {techniqueFailed && ' Until then, a scan has to name its techniques instead of choosing all, default, or a tag.'}
+          <div aria-live="polite">
+            {failures.length > 0 && (
+              // Focusable so the list can be scrolled from the keyboard.
+              <ul aria-label="Saved entries that couldn't be restored" className={styles.failureList} tabIndex={0}>
+                {failures.map((failure) => (
+                  <li key={`${failure.kind}/${failure.name ?? ''}`}>{describeFailure(failure)}</li>
+                ))}
+              </ul>
+            )}
+            {listFailed && <div>The list of saved entries couldn&apos;t be loaded. Trying again.</div>}
+          </div>
+        </MessageBarBody>
+      </MessageBar>
+    )
+  }
   return (
     <MessageBar intent="warning">
       <MessageBarBody>