Score: persist full versioned ScoringExpectation (Phase 3, PR A)

frontend/src/components/Scenarios/ScenarioRunPage.test.tsx

@@ -473,7 +473,7 @@ describe('ScenarioRunPage', () => {
       'data-location',
       `/scanner-history/${SCENARIO_RESULT_ID}/attack-result-1`,
     ))
-    const dialog = await screen.findByRole('dialog', { name: 'attack-technique' })
+    const dialog = await screen.findByRole('dialog', { name: 'attack-technique', hidden: true })
     await user.click(within(dialog).getByRole('button', { name: 'Close', hidden: true }))
 
     await waitFor(() => expect(detailsRow).toHaveFocus())

pyrit/memory/alembic/versions/1b3d5f7a9c2e_persist_scored_expectation.py

@@ -0,0 +1,177 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""
+Persist the full scoring expectation instead of a bare objective.
+
+``ScoreEntries.objective`` held only the objective string a score was judged against.
+``ScoreEntries.scored_expectation`` records the complete versioned expectation (objective
+plus any typed conditions), so a persisted score keeps the whole of what it was scored
+for. On upgrade the legacy objective is folded into an objective-only expectation; on
+downgrade only the objective survives and typed conditions are dropped.
+
+Revision ID: 1b3d5f7a9c2e
+Revises: 0f2e4d6c8b1a
+Create Date: 2026-09-03 10:00:00.000000
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Sequence  # noqa: TC003
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "1b3d5f7a9c2e"
+down_revision: str | None = "0f2e4d6c8b1a"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+logger = logging.getLogger(__name__)
+
+#: Version stamped onto every backfilled expectation.
+_SCHEMA_VERSION = 1
+
+#: Rows per page so a large score table migrates in bounded keyset batches, not one statement.
+_BACKFILL_BATCH_SIZE = 500
+
+
+def upgrade() -> None:
+    """Add ``scored_expectation``, fold the legacy objective into it, then drop ``objective``."""
+    with op.batch_alter_table("ScoreEntries") as batch_op:
+        batch_op.add_column(sa.Column("scored_expectation", sa.JSON(), nullable=True))
+
+    _backfill_scored_expectation()
+
+    with op.batch_alter_table("ScoreEntries") as batch_op:
+        batch_op.drop_column("objective")
+
+
+def downgrade() -> None:
+    """
+    Re-add ``objective``, recover it from ``scored_expectation``, then drop the expectation.
+
+    This is lossy by design: only the expectation's objective survives. Typed conditions
+    have no column in the old schema and are dropped.
+    """
+    with op.batch_alter_table("ScoreEntries") as batch_op:
+        batch_op.add_column(sa.Column("objective", sa.String(), nullable=True))
+
+    _backfill_objective()
+
+    with op.batch_alter_table("ScoreEntries") as batch_op:
+        batch_op.drop_column("scored_expectation")
+
+
+def _backfill_scored_expectation() -> None:
+    """
+    Fold every non-null legacy objective into an objective-only versioned expectation.
+
+    Rows are read a page at a time, keyed on ``id``, so a large score table is never pulled
+    into memory at once. Scores with no objective keep a NULL expectation.
+    """
+    connection = op.get_bind()
+    score_entries = sa.table(
+        "ScoreEntries",
+        sa.column("id"),
+        sa.column("objective"),
+        sa.column("scored_expectation"),
+    )
+    statement = sa.text('UPDATE "ScoreEntries" SET scored_expectation = :scored_expectation WHERE id = :score_id')
+
+    last_id = None
+    while True:
+        conditions = [
+            score_entries.c.objective.isnot(None),
+            score_entries.c.scored_expectation.is_(None),
+        ]
+        if last_id is not None:
+            conditions.append(score_entries.c.id > last_id)
+        rows = connection.execute(
+            sa.select(score_entries.c.id, score_entries.c.objective)
+            .where(*conditions)
+            .order_by(score_entries.c.id)
+            .limit(_BACKFILL_BATCH_SIZE)
+        ).fetchall()
+        if not rows:
+            return
+        last_id = rows[-1][0]
+
+        updates = [
+            {
+                "score_id": score_id,
+                "scored_expectation": json.dumps(
+                    {"schema_version": _SCHEMA_VERSION, "objective": objective, "conditions": []}
+                ),
+            }
+            for score_id, objective in rows
+        ]
+        connection.execute(statement, updates)
+
+
+def _backfill_objective() -> None:
+    """
+    Recover the objective string from every stored expectation.
+
+    Typed conditions cannot be represented by the old ``objective`` column and are dropped.
+    Rows are read a page at a time, keyed on ``id``.
+    """
+    connection = op.get_bind()
+    score_entries = sa.table(
+        "ScoreEntries",
+        sa.column("id"),
+        sa.column("objective"),
+        sa.column("scored_expectation"),
+    )
+    statement = sa.text('UPDATE "ScoreEntries" SET objective = :objective WHERE id = :score_id')
+
+    last_id = None
+    while True:
+        conditions = [
+            score_entries.c.scored_expectation.isnot(None),
+            score_entries.c.objective.is_(None),
+        ]
+        if last_id is not None:
+            conditions.append(score_entries.c.id > last_id)
+        rows = connection.execute(
+            sa.select(score_entries.c.id, score_entries.c.scored_expectation)
+            .where(*conditions)
+            .order_by(score_entries.c.id)
+            .limit(_BACKFILL_BATCH_SIZE)
+        ).fetchall()
+        if not rows:
+            return
+        last_id = rows[-1][0]
+
+        updates = []
+        for score_id, scored_expectation in rows:
+            objective = _extract_objective(scored_expectation)
+            if objective is None:
+                continue
+            updates.append({"score_id": score_id, "objective": objective})
+        if updates:
+            connection.execute(statement, updates)
+
+
+def _extract_objective(scored_expectation: object) -> str | None:
+    """
+    Read the objective out of a stored expectation, tolerating either a dict or JSON text.
+
+    Args:
+        scored_expectation (object): The stored ``scored_expectation`` value.
+
+    Returns:
+        str | None: The objective string, or ``None`` when absent or unparsable.
+    """
+    if isinstance(scored_expectation, str):
+        try:
+            scored_expectation = json.loads(scored_expectation)
+        except (ValueError, TypeError):
+            return None
+    if isinstance(scored_expectation, dict):
+        objective = scored_expectation.get("objective")
+        return objective if isinstance(objective, str) else None
+    return None

pyrit/memory/memory_models.py

@@ -63,6 +63,7 @@
     ScorerEvaluationIdentifier,
     ScorerIdentifier,
     ScoreStatus,
+    ScoringExpectation,
     Seed,
     SeedIdentifier,
     SeedObjective,
@@ -1139,7 +1140,9 @@ class ScoreEntry(Base):
     )
     prompt_request_response_id = mapped_column(CustomUUID, ForeignKey(f"{PromptMemoryEntry.__tablename__}.id"))
     timestamp = mapped_column(UTCDateTime, nullable=False)
-    objective = mapped_column(String, nullable=True)
+    # The full, versioned expectation this score was judged against (objective + conditions).
+    # Supersedes the legacy ``objective`` column.
+    scored_expectation: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
     # Version of PyRIT used when this score was created
     # Nullable for backwards compatibility with existing databases
     pyrit_version = mapped_column(String, nullable=True)
@@ -1152,7 +1155,7 @@ def __init__(self, *, entry: Score) -> None:
         Args:
             entry (Score): The score object to convert into a database entry.
         """
-        entry = Score.model_validate(entry.model_dump())
+        entry = Score.model_validate(entry.model_dump(exclude={"objective"}))
         self.id = entry.id
         self.score_value = entry.score_value
         self.score_value_description = entry.score_value_description
@@ -1176,7 +1179,7 @@ def __init__(self, *, entry: Score) -> None:
         self.scorer_identifier_hash = normalized_scorer.hash if normalized_scorer else None
         self.prompt_request_response_id = entry.message_piece_id if entry.message_piece_id else None
         self.timestamp = entry.timestamp
-        self.objective = entry.objective
+        self.scored_expectation = entry.scored_expectation.model_dump(mode="json") if entry.scored_expectation else None
         self.pyrit_version = pyrit.__version__
 
     def get_score(self) -> Score:
@@ -1207,7 +1210,11 @@ def get_score(self) -> Score:
             message_piece_id=self.prompt_request_response_id,
             scorable=scorable_from_dict(self.scorable) if self.scorable else None,
             timestamp=self.timestamp,
-            objective=self.objective,
+            scored_expectation=(
+                ScoringExpectation.model_validate_persisted(self.scored_expectation)
+                if self.scored_expectation is not None
+                else None
+            ),
         )
 
     def to_dict(self) -> dict[str, Any]:
@@ -1231,7 +1238,8 @@ def to_dict(self) -> dict[str, Any]:
             "scorable_content_id": str(self.scorable_content_id) if self.scorable_content_id else None,
             "prompt_request_response_id": str(self.prompt_request_response_id),
             "timestamp": self.timestamp.isoformat() if self.timestamp else None,
-            "objective": self.objective,
+            "scored_expectation": self.scored_expectation,
+            "objective": self.scored_expectation.get("objective") if self.scored_expectation else None,
         }
 
 

pyrit/models/__init__.py

@@ -144,6 +144,7 @@
         UndeterminedScoreError,
         UnvalidatedScore,
         scorable_from_dict,
+        scoring_expectation_fingerprint,
     )
     from pyrit.models.seeds import (
         AttackSeedGroup,
@@ -320,6 +321,7 @@
     "read_usage_int": "pyrit.models.target",
     "read_usage_value": "pyrit.models.target",
     "scorable_from_dict": "pyrit.models.score",
+    "scoring_expectation_fingerprint": "pyrit.models.score",
     "validate_registry_name": "pyrit.models.identifiers",
     "project_behavioral_identity": "pyrit.models.identifiers",
     "RetryEvent": "pyrit.models.retry_event",

pyrit/models/score/__init__.py

@@ -16,7 +16,10 @@
 
 if TYPE_CHECKING:
     from pyrit.models.score.condition import Condition, MatchesObjective
-    from pyrit.models.score.expectation import ScoringExpectation
+    from pyrit.models.score.expectation import (
+        ScoringExpectation,
+        scoring_expectation_fingerprint,
+    )
     from pyrit.models.score.scorable import (
         ContentEntryScorable,
         ContentScorable,
@@ -50,6 +53,7 @@
     "UndeterminedScoreError": "pyrit.models.score.score",
     "UnvalidatedScore": "pyrit.models.score.score",
     "scorable_from_dict": "pyrit.models.score.scorable",
+    "scoring_expectation_fingerprint": "pyrit.models.score.expectation",
 }
 
 __all__ = list(_LAZY_EXPORTS)