diff --git a/ENV_VARIABLES.md b/ENV_VARIABLES.md index bf4858ad1..66d8e017a 100644 --- a/ENV_VARIABLES.md +++ b/ENV_VARIABLES.md @@ -14,6 +14,31 @@ and source-mode CLIs fail before making an API request when no URL is configured Help, version, and CLI updates do not require a Scope API URL. The CLI no longer derives a localhost URL from `SCOPE_API_PORT` or uses `SCOPE_DEFAULT_API_URL`. +## Taxonome post-processing + +### TAXONOME_MODEL +**Default:** `claude-haiku-4.5` +**Type:** model identifier + +Model used by the post-processor's lazy Taxonome prompt executor. The worker +does not resolve credentials or initialize a Copilot SDK transport for +deterministic-only taxonomies. Prompt execution uses the taxonomy engine's +isolated, tool-free Copilot transport and normal credential resolution. This +setting is independent from the coding run's model and reasoning effort. +Explicit model IDs are checked against `CopilotClient.listModels()` before +session creation and fail clearly when unavailable; `auto` remains available +as an explicit provider-resolved override. The post-processor deliberately +omits reasoning effort so a model that does not support configurable effort +remains valid. + +### TAXONOMY_RESULT_MAX_BYTES +**Default:** `10485760` (10 MiB) +**Type:** positive integer bytes + +Maximum uncompressed Taxonome result size returned by the API's validated +exact-run JSON view. The API rejects a larger persisted reference before +downloading the blob and also enforces the limit while decompressing it. + ## Docker Development ### NPM_CONFIG_REGISTRY @@ -467,21 +492,21 @@ a fallback or default. ## Judge Strategy Configuration ### JUDGE_MODEL -**Default:** `gpt-5.4-mini` +**Default:** `auto` **Type:** string -Model used by the judge to evaluate agent output against criteria. Defaults to -`gpt-5.4-mini` (set in code, docker-compose, and the K8s manifest). Override via -`JUDGE_MODEL` (e.g. in `.env`) to use a different model. +Model used by the judge to evaluate agent output against criteria. `auto` +delegates selection to the Copilot provider so local environments do not pin a +model unavailable to the active credential. Override via `JUDGE_MODEL` (e.g. in +`.env`) when a deployment requires an exact model. ### FEEDBACK_MODEL -**Default:** `gpt-5.4-mini` +**Default:** `auto` **Type:** string Model used by the feedback generator that produces actionable feedback for the -coding agent between iterations. Defaults to `gpt-5.4-mini` (set in code, -docker-compose, and the K8s manifest). Override via `FEEDBACK_MODEL` (e.g. in -`.env`) to use a different model. +coding agent between iterations. `auto` uses the Copilot provider's available +model routing. Override via `FEEDBACK_MODEL` (e.g. in `.env`) to pin a model. ### JUDGE_STRATEGY **Default:** `bundled` @@ -908,10 +933,13 @@ FEEDBACK_DESCENDANT_GUARD=false Azure Storage Queue name for report generation jobs. The API enqueues messages here when a report is requested; the report-generator worker polls this queue. ### REPORT_MODEL -**Default:** `gpt-5.4-mini` +**Default:** `auto` **Type:** string -The LLM model used by the report-generator worker (via the Copilot SDK) to generate run analysis reports. Examples: `gpt-5.4-mini`, `gpt-4.1`, `gpt-4o`, `claude-sonnet-4`. +The LLM model used by the report-generator worker (via the Copilot SDK) to +generate run analysis reports. `auto` avoids pinning local Compose to a model +outside the active credential's catalog. Set an exact provider-supported model +only when deterministic deployment policy requires it. ### SCOPE_MT_API_URL **Default:** `http://localhost:3001` (local), `http://api:80` (Docker) @@ -971,15 +999,16 @@ valid `GIT_COMMIT` and `GIT_BRANCH` build inputs still take precedence. Azure Storage Queue name for Experiment readout jobs. The coordinator enqueues an exact execution message containing `projectId`, `experimentId`, `experimentExecutionId`, handler identity/version, claim token, and execution -version only after `pp-observations` is done for every exact request/run pair. +version only after `pp-taxonomy` is done for every exact request/run pair. ### READOUT_MODEL -**Default:** `gpt-5.4` +**Default:** `auto` **Type:** string The Copilot SDK model used by the dedicated `pp-readout` worker to synthesize a -schema-validated structured Experiment readout. Markdown is derived from the -validated structured object. +schema-validated structured Experiment readout. `auto` delegates selection to +the provider so local Compose does not pin an unavailable model. Markdown is +derived from the validated structured object. ### READOUT_SESSION_TIMEOUT_MS **Default:** `300000` (5 minutes) diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 64d226e3d..307d9253e 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -28,6 +28,7 @@ COPY packages/db-migrations/tsconfig.json ./packages/db-migrations/ COPY packages/db-migrations/src ./packages/db-migrations/src/ RUN pnpm --filter db-migrations build +COPY packages/ax-taxonomy/taxonomy ./packages/ax-taxonomy/taxonomy/ COPY apps/api/tsconfig.json ./apps/api/ COPY apps/api/src ./apps/api/src/ @@ -65,6 +66,7 @@ RUN NPM_CONFIG_REGISTRY="${NPM_CONFIG_REGISTRY}" pnpm install --frozen-lockfile COPY --from=builder /app/packages/shared/dist ./packages/shared/dist/ COPY --from=builder /app/packages/telemetry/dist ./packages/telemetry/dist/ COPY --from=builder /app/packages/db-migrations/dist ./packages/db-migrations/dist/ +COPY --from=builder /app/packages/ax-taxonomy/taxonomy ./packages/ax-taxonomy/taxonomy/ COPY --from=builder /app/apps/api/dist ./apps/api/dist/ WORKDIR /app/apps/api diff --git a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap index fe9de087a..7ba9f7220 100644 --- a/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap +++ b/apps/api/src/__snapshots__/openapi-snapshot.test.ts.snap @@ -625,6 +625,31 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, + "CopyBuiltinTaxonomyRevisionInput": { + "additionalProperties": false, + "properties": { + "creator": { + "type": "string", + }, + "description": { + "maxLength": 4096, + "type": "string", + }, + "id": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$", + "type": "string", + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "name", + ], + "type": "object", + }, "CreateAgentInput": { "properties": { "_id": { @@ -771,6 +796,10 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "default": [], "items": { "properties": { + "agentVersion": { + "minLength": 1, + "type": "string", + }, "agentsMd": { "type": "string", }, @@ -866,6 +895,10 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "maxItems": 250000, "type": "array", }, + "taxonomyRevisionId": { + "minLength": 1, + "type": "string", + }, }, "required": [ "mode", @@ -1317,6 +1350,110 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, + "CreateTaxonomyElementInput": { + "additionalProperties": false, + "properties": { + "content": { + "$ref": "#/components/schemas/PortableTaxonomyElement", + }, + "creator": { + "type": "string", + }, + "description": { + "maxLength": 4096, + "type": "string", + }, + "id": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$", + "type": "string", + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string", + }, + "sourceType": { + "default": "user", + "enum": [ + "builtin", + "user", + ], + "type": "string", + }, + }, + "required": [ + "name", + "content", + ], + "type": "object", + }, + "CreateTaxonomyElementRevisionInput": { + "additionalProperties": false, + "properties": { + "content": { + "$ref": "#/components/schemas/PortableTaxonomyElement", + }, + "creator": { + "type": "string", + }, + }, + "required": [ + "content", + ], + "type": "object", + }, + "CreateTaxonomyInput": { + "additionalProperties": false, + "properties": { + "content": { + "$ref": "#/components/schemas/PortableTaxonomy", + }, + "creator": { + "type": "string", + }, + "description": { + "maxLength": 4096, + "type": "string", + }, + "id": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$", + "type": "string", + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string", + }, + "sourceType": { + "default": "user", + "enum": [ + "builtin", + "user", + ], + "type": "string", + }, + }, + "required": [ + "name", + "content", + ], + "type": "object", + }, + "CreateTaxonomyRevisionInput": { + "additionalProperties": false, + "properties": { + "content": { + "$ref": "#/components/schemas/PortableTaxonomy", + }, + "creator": { + "type": "string", + }, + }, + "required": [ + "content", + ], + "type": "object", + }, "CriteriaGraph": { "properties": { "edges": { @@ -1795,6 +1932,166 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "type": "object", }, + "requestCount": { + "minimum": 0, + "type": "integer", + }, + "requests": { + "items": { + "properties": { + "agentVersion": { + "type": "string", + }, + "attemptNumber": { + "exclusiveMinimum": 0, + "type": "integer", + }, + "criteria": { + "items": { + "type": "string", + }, + "type": "array", + }, + "extensions": { + "items": { + "type": "string", + }, + "type": "array", + }, + "lastHeartbeatAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "mcpServers": { + "items": { + "type": "string", + }, + "type": "array", + }, + "model": { + "type": "string", + }, + "os": { + "properties": { + "arch": { + "type": "string", + }, + "platform": { + "type": "string", + }, + "release": { + "type": "string", + }, + }, + "required": [ + "platform", + "release", + "arch", + ], + "type": "object", + }, + "outcome": { + "type": "string", + }, + "priority": { + "type": "number", + }, + "profileId": { + "type": "string", + }, + "reasoningEffort": { + "type": "string", + }, + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + "skillRevisions": { + "items": { + "type": "string", + }, + "type": "array", + }, + "startedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "status": { + "type": "string", + }, + "task": { + "type": "string", + }, + "taskPromptId": { + "type": "string", + }, + "taxonomyHandler": { + "additionalProperties": false, + "properties": { + "error": { + "type": "string", + }, + "resultAvailable": { + "type": "boolean", + }, + "status": { + "enum": [ + "pending", + "queued", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "status", + "resultAvailable", + ], + "type": "object", + }, + "worker": { + "properties": { + "instanceId": { + "type": "string", + }, + "podName": { + "type": "string", + }, + }, + "required": [ + "instanceId", + ], + "type": "object", + }, + "workerType": { + "type": "string", + }, + }, + "required": [ + "requestId", + "runId", + ], + "type": "object", + }, + "type": "array", + }, "startedAt": { "format": "date-time", "type": [ @@ -1828,11 +2125,16 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "null", ], }, + "version": { + "type": "integer", + }, }, "required": [ "experimentExecutionId", "attemptNumber", "readiness", + "requestCount", + "requests", "blockers", "progress", "reportTemplates", @@ -2290,12 +2592,172 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "type": "object", }, - "startedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], + "requestCount": { + "minimum": 0, + "type": "integer", + }, + "requests": { + "items": { + "properties": { + "agentVersion": { + "type": "string", + }, + "attemptNumber": { + "exclusiveMinimum": 0, + "type": "integer", + }, + "criteria": { + "items": { + "type": "string", + }, + "type": "array", + }, + "extensions": { + "items": { + "type": "string", + }, + "type": "array", + }, + "lastHeartbeatAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "mcpServers": { + "items": { + "type": "string", + }, + "type": "array", + }, + "model": { + "type": "string", + }, + "os": { + "properties": { + "arch": { + "type": "string", + }, + "platform": { + "type": "string", + }, + "release": { + "type": "string", + }, + }, + "required": [ + "platform", + "release", + "arch", + ], + "type": "object", + }, + "outcome": { + "type": "string", + }, + "priority": { + "type": "number", + }, + "profileId": { + "type": "string", + }, + "reasoningEffort": { + "type": "string", + }, + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + "skillRevisions": { + "items": { + "type": "string", + }, + "type": "array", + }, + "startedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "status": { + "type": "string", + }, + "task": { + "type": "string", + }, + "taskPromptId": { + "type": "string", + }, + "taxonomyHandler": { + "additionalProperties": false, + "properties": { + "error": { + "type": "string", + }, + "resultAvailable": { + "type": "boolean", + }, + "status": { + "enum": [ + "pending", + "queued", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "status", + "resultAvailable", + ], + "type": "object", + }, + "worker": { + "properties": { + "instanceId": { + "type": "string", + }, + "podName": { + "type": "string", + }, + }, + "required": [ + "instanceId", + ], + "type": "object", + }, + "workerType": { + "type": "string", + }, + }, + "required": [ + "requestId", + "runId", + ], + "type": "object", + }, + "type": "array", + }, + "startedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], }, "updatedAt": { "format": "date-time", @@ -2304,11 +2766,16 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "null", ], }, + "version": { + "type": "integer", + }, }, "required": [ "experimentExecutionId", "attemptNumber", "readiness", + "requestCount", + "requests", "blockers", "progress", "reportTemplates", @@ -2486,6 +2953,25 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "revisionNumber": { "type": "integer", }, + "taxonomy": { + "properties": { + "contentSha256": { + "type": "string", + }, + "ref": { + "type": "string", + }, + "taxonomyRevisionId": { + "type": "string", + }, + }, + "required": [ + "taxonomyRevisionId", + "ref", + "contentSha256", + ], + "type": "object", + }, }, "required": [ "experimentRevisionId", @@ -2816,6 +3302,39 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "type": "object", }, + "ImportTaxonomyYamlInput": { + "additionalProperties": false, + "properties": { + "creator": { + "type": "string", + }, + "description": { + "maxLength": 4096, + "type": "string", + }, + "elements": { + "additionalProperties": { + "type": "string", + }, + "default": {}, + "type": "object", + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string", + }, + "taxonomyYaml": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "name", + "taxonomyYaml", + ], + "type": "object", + }, "InsightReference": { "properties": { "insightId": { @@ -3247,64 +3766,254 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, - "ProfileResponse": { + "PortableTaxonomy": { + "additionalProperties": false, "properties": { - "_id": { - "type": "string", + "elements": { + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + }, + }, + "required": [ + "path", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "ref": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*@r[1-9]\\d*$", + "type": "string", + }, + }, + "required": [ + "ref", + ], + "type": "object", + }, + ], + }, + "maxItems": 200, + "minItems": 1, + "type": "array", }, - "createdAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], + "id": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$", + "type": "string", }, - "deletedAt": { - "format": "date-time", - "type": [ - "string", - "null", + "schema_version": { + "enum": [ + "scope/taxonomy/v1", ], - }, - "description": { "type": "string", }, - "latestVersion": { - "type": "number", - }, - "name": { - "type": "string", + "surfaces": { + "items": { + "additionalProperties": false, + "properties": { + "description": { + "minLength": 1, + "type": "string", + }, + "id": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$", + "type": "string", + }, + "label": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "id", + "label", + ], + "type": "object", + }, + "maxItems": 100, + "type": "array", }, - "projectId": { + "version": { + "pattern": "^\\d+\\.\\d+\\.\\d+(?:-[0-9A-Za-z.-]+)?$", "type": "string", }, - "updatedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, }, "required": [ - "_id", - "name", - "latestVersion", - "createdAt", - "projectId", + "schema_version", + "id", + "version", + "surfaces", + "elements", ], "type": "object", }, - "ProfileVersionResponse": { + "PortableTaxonomyElement": { + "additionalProperties": false, "properties": { - "_id": { + "category_id": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$", "type": "string", }, - "agentVersion": { + "description": { + "minLength": 1, "type": "string", }, - "createdAt": { - "format": "date-time", + "id": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$", + "type": "string", + }, + "implementation": { + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "prompt": { + "minLength": 1, + "type": "string", + }, + "type": { + "enum": [ + "prompt", + ], + "type": "string", + }, + }, + "required": [ + "type", + "prompt", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "config": {}, + "handler": { + "minLength": 1, + "type": "string", + }, + "type": { + "enum": [ + "deterministic", + ], + "type": "string", + }, + }, + "required": [ + "type", + "handler", + ], + "type": "object", + }, + ], + }, + "label": { + "minLength": 1, + "type": "string", + }, + "output": { + "additionalProperties": false, + "properties": { + "constraints": { + "items": {}, + "type": "array", + }, + "evidence_roles": { + "additionalProperties": {}, + "type": "object", + }, + }, + "required": [ + "evidence_roles", + ], + "type": "object", + }, + "preconditions": { + "additionalProperties": { + "type": "boolean", + }, + "type": "object", + }, + "schema_version": { + "enum": [ + "scope/taxonomy/v1", + ], + "type": "string", + }, + }, + "required": [ + "id", + "label", + "implementation", + "preconditions", + "output", + ], + "type": "object", + }, + "ProfileResponse": { + "properties": { + "_id": { + "type": "string", + }, + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "deletedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "description": { + "type": "string", + }, + "latestVersion": { + "type": "number", + }, + "name": { + "type": "string", + }, + "projectId": { + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "_id", + "name", + "latestVersion", + "createdAt", + "projectId", + ], + "type": "object", + }, + "ProfileVersionResponse": { + "properties": { + "_id": { + "type": "string", + }, + "agentVersion": { + "type": "string", + }, + "createdAt": { + "format": "date-time", "type": [ "string", "null", @@ -4211,6 +4920,39 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "taskPromptId": { "type": "string", }, + "taxonomyHandler": { + "additionalProperties": false, + "properties": { + "error": { + "type": "string", + }, + "resultAvailable": { + "type": "boolean", + }, + "status": { + "enum": [ + "pending", + "queued", + "processing", + "done", + "failed", + ], + "type": "string", + }, + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + "required": [ + "status", + "resultAvailable", + ], + "type": "object", + }, "updatedAt": { "format": "date-time", "type": [ @@ -4242,6 +4984,27 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "type": "object", }, + "ResolvedTaxonomyElementResponse": { + "additionalProperties": false, + "properties": { + "position": { + "minimum": 0, + "type": "integer", + }, + "revision": { + "$ref": "#/components/schemas/TaxonomyElementRevisionResponse", + }, + "root": { + "$ref": "#/components/schemas/TaxonomyElementResponse", + }, + }, + "required": [ + "position", + "root", + "revision", + ], + "type": "object", + }, "ResourceBinding": { "properties": { "params": { @@ -4837,6 +5600,34 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "string", }, + "taxonomyEvaluation": { + "additionalProperties": false, + "properties": { + "contentSha256": { + "pattern": "^[a-f0-9]{64}$", + "type": "string", + }, + "schemaVersion": { + "minLength": 1, + "type": "string", + }, + "sizeBytes": { + "minimum": 0, + "type": "integer", + }, + "url": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "url", + "schemaVersion", + "contentSha256", + "sizeBytes", + ], + "type": "object", + }, "tokenUsage": { "$ref": "#/components/schemas/TokenUsage", }, @@ -5235,412 +6026,1615 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, - "TokenUsage": { + "TaxonomyDetailResponse": { + "additionalProperties": false, "properties": { - "completionTokens": { - "type": "number", + "elements": { + "items": { + "$ref": "#/components/schemas/ResolvedTaxonomyElementResponse", + }, + "type": "array", }, - "promptTokens": { - "type": "number", + "revisions": { + "items": { + "$ref": "#/components/schemas/TaxonomyRevisionSummaryResponse", + }, + "type": "array", }, - "totalTokens": { - "type": "number", + "selectedRevision": { + "$ref": "#/components/schemas/TaxonomyRevisionResponse", + }, + "taxonomy": { + "$ref": "#/components/schemas/TaxonomyResponse", }, }, "required": [ - "promptTokens", - "completionTokens", - "totalTokens", + "taxonomy", + "selectedRevision", + "revisions", + "elements", ], "type": "object", }, - "TriggerReportsInput": { + "TaxonomyElementResponse": { "properties": { - "requestId": { + "_id": { "type": "string", }, - "runId": { - "type": "string", + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], }, - }, - "required": [ - "requestId", - ], - "type": "object", - }, - "UpdateAgentInput": { - "properties": { - "available": { - "type": "boolean", + "creator": { + "type": "string", }, - "capabilities": { - "$ref": "#/components/schemas/AgentCapabilities", + "deletedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], }, - "defaultModel": { + "description": { "type": "string", }, - "description": { + "id": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$", "type": "string", }, - "modelProvider": { + "latestRevision": { + "exclusiveMinimum": 0, + "type": "integer", + }, + "latestRevisionId": { "type": "string", }, "name": { "type": "string", }, - "supportedModels": { - "items": { - "type": "string", - }, - "type": "array", - }, - }, - "type": "object", - }, - "UpdateCodebaseInput": { - "properties": { - "defaultBranch": { + "projectId": { "type": "string", }, - "description": { - "type": "string", - }, - "name": { - "minLength": 1, - "type": "string", - }, - }, - "type": "object", - }, - "UpdateCriteriaInput": { - "properties": { - "dependsOn": { - "items": { - "type": "string", - }, - "type": "array", - }, - "gates": { - "items": { - "enum": [ - "select", - "build", - "test", - "run", - "deploy", - ], - "type": "string", - }, - "type": "array", - }, - "kind": { - "enum": [ - "gate", - "observation", - ], - "type": "string", - }, - "prompt": { - "type": "string", + "revisionCounter": { + "minimum": 0, + "type": "integer", }, - "subject": { + "sourceType": { "enum": [ - "run", - "iteration", + "builtin", + "user", ], "type": "string", }, - "taxonomyElementId": { - "enum": [ - "dimension:idiomatic-use", - "dimension:dependency-currency", - "dimension:configuration-correctness", + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", ], - "type": "string", }, }, + "required": [ + "_id", + "projectId", + "id", + "name", + "sourceType", + "revisionCounter", + "createdAt", + ], "type": "object", }, - "UpdateExtensionInput": { - "properties": { - "description": { - "type": "string", - }, - "name": { - "type": "string", + "TaxonomyElementRevisionResponse": { + "allOf": [ + { + "$ref": "#/components/schemas/PortableTaxonomyElement", }, - }, - "type": "object", - }, - "UpdateFeatureFlagInput": { - "properties": { - "enabled": { - "type": "boolean", + { + "additionalProperties": false, + "default": undefined, + "properties": { + "_id": { + "type": "string", + }, + "contentSha256": { + "maxLength": 64, + "minLength": 64, + "type": "string", + }, + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "creator": { + "type": "string", + }, + "elementId": { + "type": "string", + }, + "projectId": { + "type": "string", + }, + "ref": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*@r[1-9]\\d*$", + "type": "string", + }, + "revision": { + "exclusiveMinimum": 0, + "type": "integer", + }, + "sourceType": { + "enum": [ + "builtin", + "user", + ], + "type": "string", + }, + }, + "required": [ + "_id", + "elementId", + "projectId", + "sourceType", + "revision", + "ref", + "contentSha256", + "createdAt", + ], + "type": "object", }, - }, - "required": [ - "enabled", ], - "type": "object", }, - "UpdateInsightInput": { + "TaxonomyEvaluationReference": { "properties": { - "category": { + "contentSha256": { "type": "string", }, - "description": { + "schemaVersion": { "type": "string", }, - "tags": { - "items": { - "type": "string", - }, - "type": "array", + "sizeBytes": { + "minimum": 0, + "type": "integer", }, - "title": { + "url": { "type": "string", }, }, + "required": [ + "url", + "schemaVersion", + "contentSha256", + "sizeBytes", + ], "type": "object", }, - "UpdateMcpServerInput": { - "properties": { - "args": { - "items": { - "type": "string", + "TaxonomyListItemResponse": { + "allOf": [ + { + "$ref": "#/components/schemas/TaxonomyResponse", + }, + { + "default": undefined, + "properties": { + "elementCount": { + "minimum": 0, + "type": "integer", + }, + "ref": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*@r[1-9]\\d*$", + "type": "string", + }, + "version": { + "type": "string", + }, }, - "type": "array", + "required": [ + "ref", + "version", + "elementCount", + ], + "type": "object", }, - "command": { + ], + }, + "TaxonomyReadoutReference": { + "properties": { + "error": { "type": "string", }, - "description": { + "experimentExecutionId": { "type": "string", }, - "env": { - "additionalProperties": { - "type": "string", - }, - "type": "object", - }, - "headers": { - "items": { - "$ref": "#/components/schemas/McpServerHeader", - }, - "type": "array", - }, - "name": { + "jsonUrl": { "type": "string", }, - "sessionMode": { - "enum": [ - "stateful", - "stateless", - ], + "markdownUrl": { "type": "string", }, - "type": { + "status": { "enum": [ - "sse", - "http", - "stdio", + "processing", + "done", + "failed", ], "type": "string", }, - "url": { + "taxonomyRevisionId": { "type": "string", }, - "version": { - "type": "string", + "updatedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], }, }, + "required": [ + "status", + "experimentExecutionId", + "taxonomyRevisionId", + "updatedAt", + ], "type": "object", }, - "UpdateProfileIdentity": { + "TaxonomyResponse": { "properties": { - "description": { - "maxLength": 512, + "_id": { "type": "string", }, - "name": { - "maxLength": 128, - "minLength": 1, + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "creator": { "type": "string", }, - }, - "type": "object", - }, - "UpdateProjectInput": { - "properties": { + "deletedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, "description": { "type": "string", }, + "id": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$", + "type": "string", + }, + "latestRevision": { + "exclusiveMinimum": 0, + "type": "integer", + }, + "latestRevisionId": { + "type": "string", + }, "name": { - "minLength": 1, "type": "string", }, - }, - "type": "object", - }, - "UpdatePromptFeatureInput": { - "properties": { - "prompt": { + "projectId": { "type": "string", }, - }, - "type": "object", - }, - "UpdateReportTemplateInput": { - "properties": { - "description": { - "type": "string", + "revisionCounter": { + "minimum": 0, + "type": "integer", }, - "enabled": { - "type": "boolean", + "sourceType": { + "enum": [ + "builtin", + "user", + ], + "type": "string", }, - "model": { + "updatedAt": { + "format": "date-time", "type": [ "string", "null", ], }, - "name": { - "type": "string", + }, + "required": [ + "_id", + "projectId", + "id", + "name", + "sourceType", + "revisionCounter", + "createdAt", + ], + "type": "object", + }, + "TaxonomyRevisionResponse": { + "allOf": [ + { + "$ref": "#/components/schemas/PortableTaxonomy", }, - "systemPrompt": { - "allOf": [ - { - "$ref": "#/components/schemas/ReportTemplateSystemPrompt", + { + "additionalProperties": false, + "default": undefined, + "properties": { + "_id": { + "type": "string", }, - { + "contentSha256": { + "maxLength": 64, + "minLength": 64, + "type": "string", + }, + "createdAt": { + "format": "date-time", "type": [ - "object", + "string", "null", ], }, + "creator": { + "type": "string", + }, + "projectId": { + "type": "string", + }, + "ref": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*@r[1-9]\\d*$", + "type": "string", + }, + "revision": { + "exclusiveMinimum": 0, + "type": "integer", + }, + "sourceType": { + "enum": [ + "builtin", + "user", + ], + "type": "string", + }, + "taxonomyId": { + "type": "string", + }, + }, + "required": [ + "_id", + "taxonomyId", + "projectId", + "sourceType", + "revision", + "ref", + "contentSha256", + "createdAt", ], + "type": "object", }, - "timeoutMs": { - "exclusiveMinimum": 0, - "type": [ - "integer", - "null", - ], - }, - "trigger": { - "$ref": "#/components/schemas/ReportTrigger", - }, - "userPrompt": { - "type": "string", - }, - }, - "type": "object", + ], }, - "UpdateResourceInput": { + "TaxonomyRevisionSummaryResponse": { + "additionalProperties": false, "properties": { - "description": { + "_id": { "type": "string", }, - "name": { - "minLength": 1, + "contentSha256": { + "maxLength": 64, + "minLength": 64, "type": "string", }, - }, - "type": "object", - }, - "UserMeResponse": { - "properties": { - "displayName": { - "type": "string", + "createdAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], }, - "email": { + "creator": { "type": "string", }, - "id": { - "type": "string", + "elementCount": { + "minimum": 0, + "type": "integer", }, - "idp": { - "type": "string", + "isLatest": { + "type": "boolean", }, - "idpTenant": { + "ref": { + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*@r[1-9]\\d*$", "type": "string", }, - "role": { - "type": "string", + "revision": { + "exclusiveMinimum": 0, + "type": "integer", }, - }, - "required": [ - "id", - ], - "type": "object", - }, - "ValidateKeyInput": { - "properties": { - "token": { + "version": { "type": "string", }, }, "required": [ - "token", + "_id", + "revision", + "ref", + "version", + "elementCount", + "contentSha256", + "createdAt", + "isLatest", ], "type": "object", }, - }, - "securitySchemes": { - "bearerAuth": { - "bearerFormat": "JWT", - "description": "Use the unchanged IdP access token, without the Bearer prefix.", - "scheme": "bearer", - "type": "http", - }, - }, - }, - "info": { - "description": "REST API for the Scope platform — benchmarking AI coding agents", - "title": "Scope API", - "version": "1.0.0", - }, - "openapi": "3.1.0", - "paths": { - "/about": { - "get": { - "responses": { - "200": { - "content": { - "application/json": { - "schema": { + "TaxonomyRunResultResponse": { + "additionalProperties": false, + "properties": { + "metadata": {}, + "results": { + "items": { + "oneOf": [ + { + "additionalProperties": false, "properties": { - "buildTime": { - "type": "string", - }, - "description": { - "type": "string", - }, - "environment": { - "type": "string", - }, - "name": { - "type": "string", + "element": { + "additionalProperties": false, + "properties": { + "category_id": { + "minLength": 1, + "type": "string", + }, + "id": { + "minLength": 1, + "type": "string", + }, + "label": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "id", + "label", + ], + "type": "object", }, - "version": { - "type": "string", + "implementation": { + "additionalProperties": false, + "properties": { + "attempts": { + "items": { + "additionalProperties": false, + "properties": { + "attempt": { + "anyOf": [ + { + "enum": [ + 1, + ], + "type": "number", + }, + { + "enum": [ + 2, + ], + "type": "number", + }, + ], + }, + "kind": { + "enum": [ + "initial", + "repair", + ], + "type": "string", + }, + "outcome": { + "enum": [ + "succeeded", + "invalid_output", + "execution_failed", + "cancelled", + ], + "type": "string", + }, + }, + "required": [ + "attempt", + "kind", + "outcome", + ], + "type": "object", + }, + "maxItems": 2, + "minItems": 1, + "type": "array", + }, + "handler": { + "minLength": 1, + "type": "string", + }, + "provider": { + "additionalProperties": false, + "properties": { + "entry": { + "minLength": 1, + "type": "string", + }, + "integrity": { + "pattern": "^[a-f0-9]{64}$", + "type": "string", + }, + "name": { + "minLength": 1, + "type": "string", + }, + "origin": { + "enum": [ + "builtin", + "local", + "package", + ], + "type": "string", + }, + "version": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "name", + "origin", + ], + "type": "object", + }, + "type": { + "enum": [ + "prompt", + "deterministic", + ], + "type": "string", + }, + }, + "required": [ + "type", + "attempts", + ], + "type": "object", }, - "workers": { - "items": { - "type": "string", + "input": { + "additionalProperties": false, + "properties": { + "archive_root": { + "minLength": 1, + "type": "string", + }, + "normalization": { + "enum": [ + "scope/taxonome-normalized-atif/v1", + ], + "type": "string", + }, + "request_id": { + "minLength": 1, + "type": "string", + }, + "run_id": { + "minLength": 1, + "type": "string", + }, + "session_id": { + "minLength": 1, + "type": "string", + }, + "sha256": { + "pattern": "^[a-f0-9]{64}$", + "type": "string", + }, + "trajectory_id": { + "minLength": 1, + "type": "string", + }, }, - "type": "array", + "required": [ + "normalization", + "sha256", + ], + "type": "object", }, - }, - "required": [ - "name", - "version", - "buildTime", - "environment", - "description", - "workers", - ], - "type": "object", + "result": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "observed": { + "enum": [ + true, + ], + "type": "boolean", + }, + "occurrences": { + "items": { + "additionalProperties": false, + "properties": { + "evidence": { + "items": { + "additionalProperties": false, + "properties": { + "path": { + "minLength": 1, + "type": "string", + }, + "role": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "role", + "path", + ], + "type": "object", + }, + "maxItems": 100, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "evidence", + ], + "type": "object", + }, + "maxItems": 100, + "minItems": 1, + "type": "array", + }, + "state": { + "enum": [ + "completed", + ], + "type": "string", + }, + }, + "required": [ + "state", + "observed", + "occurrences", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "observed": { + "enum": [ + false, + ], + "type": "boolean", + }, + "state": { + "enum": [ + "completed", + ], + "type": "string", + }, + }, + "required": [ + "state", + "observed", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "reason": { + "maxLength": 2000, + "minLength": 1, + "type": "string", + }, + "state": { + "enum": [ + "indeterminate", + "inapplicable", + ], + "type": "string", + }, + }, + "required": [ + "state", + "reason", + ], + "type": "object", + }, + ], + }, + "schema_version": { + "enum": [ + "scope/taxonomy-evaluation/v1", + ], + "type": "string", + }, + "status": { + "enum": [ + "succeeded", + ], + "type": "string", + }, + "taxonomy": { + "additionalProperties": false, + "properties": { + "digest": { + "pattern": "^[a-f0-9]{64}$", + "type": "string", + }, + "id": { + "minLength": 1, + "type": "string", + }, + "version": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "id", + "version", + "digest", + ], + "type": "object", + }, + }, + "required": [ + "schema_version", + "taxonomy", + "element", + "implementation", + "input", + "status", + "result", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "element": { + "additionalProperties": false, + "properties": { + "category_id": { + "minLength": 1, + "type": "string", + }, + "id": { + "minLength": 1, + "type": "string", + }, + "label": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "id", + "label", + ], + "type": "object", + }, + "error": { + "additionalProperties": false, + "properties": { + "code": { + "enum": [ + "execution_failed", + "cancelled", + "invalid_output", + "invalid_input_binding", + ], + "type": "string", + }, + "message": { + "maxLength": 2000, + "minLength": 1, + "type": "string", + }, + "source_locations": { + "items": { + "additionalProperties": false, + "properties": { + "archive_entry": { + "type": "string", + }, + "file": { + "type": "string", + }, + "generated_by": { + "type": "string", + }, + "jsonl_line": { + "exclusiveMinimum": 0, + "type": "integer", + }, + "normalized_path": { + "minLength": 1, + "type": "string", + }, + "original_path": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "normalized_path", + "original_path", + ], + "type": "object", + }, + "maxItems": 100, + "type": "array", + }, + "validation_errors": { + "items": { + "additionalProperties": false, + "properties": { + "actual_type": { + "enum": [ + "atif.step", + "atif.tool_call", + "atif.observation", + "atif.observation_result", + "atif.content_part", + "atif.image_source", + "atif.audio_source", + "scope.discovery", + "scope.input_validation_issue", + ], + "type": "string", + }, + "code": { + "enum": [ + "invalid_json", + "schema_mismatch", + "unknown_role", + "role_cardinality", + "duplicate_reference", + "invalid_jsonpath", + "unresolved_reference", + "reference_type_mismatch", + "reference_constraint_mismatch", + ], + "type": "string", + }, + "evidence_path": { + "minLength": 1, + "type": "string", + }, + "expected_type": { + "enum": [ + "atif.step", + "atif.tool_call", + "atif.observation", + "atif.observation_result", + "atif.content_part", + "atif.image_source", + "atif.audio_source", + "scope.discovery", + "scope.input_validation_issue", + ], + "type": "string", + }, + "message": { + "maxLength": 1000, + "minLength": 1, + "type": "string", + }, + "output_path": { + "minLength": 1, + "type": "string", + }, + "role": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "code", + "output_path", + ], + "type": "object", + }, + "maxItems": 100, + "type": "array", + }, + }, + "required": [ + "code", + "message", + ], + "type": "object", + }, + "implementation": { + "additionalProperties": false, + "properties": { + "attempts": { + "items": { + "additionalProperties": false, + "properties": { + "attempt": { + "anyOf": [ + { + "enum": [ + 1, + ], + "type": "number", + }, + { + "enum": [ + 2, + ], + "type": "number", + }, + ], + }, + "kind": { + "enum": [ + "initial", + "repair", + ], + "type": "string", + }, + "outcome": { + "enum": [ + "succeeded", + "invalid_output", + "execution_failed", + "cancelled", + ], + "type": "string", + }, + }, + "required": [ + "attempt", + "kind", + "outcome", + ], + "type": "object", + }, + "maxItems": 2, + "minItems": 1, + "type": "array", + }, + "handler": { + "minLength": 1, + "type": "string", + }, + "provider": { + "additionalProperties": false, + "properties": { + "entry": { + "minLength": 1, + "type": "string", + }, + "integrity": { + "pattern": "^[a-f0-9]{64}$", + "type": "string", + }, + "name": { + "minLength": 1, + "type": "string", + }, + "origin": { + "enum": [ + "builtin", + "local", + "package", + ], + "type": "string", + }, + "version": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "name", + "origin", + ], + "type": "object", + }, + "type": { + "enum": [ + "prompt", + "deterministic", + ], + "type": "string", + }, + }, + "required": [ + "type", + "attempts", + ], + "type": "object", + }, + "input": { + "additionalProperties": false, + "properties": { + "archive_root": { + "minLength": 1, + "type": "string", + }, + "normalization": { + "enum": [ + "scope/taxonome-normalized-atif/v1", + ], + "type": "string", + }, + "request_id": { + "minLength": 1, + "type": "string", + }, + "run_id": { + "minLength": 1, + "type": "string", + }, + "session_id": { + "minLength": 1, + "type": "string", + }, + "sha256": { + "pattern": "^[a-f0-9]{64}$", + "type": "string", + }, + "trajectory_id": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "normalization", + "sha256", + ], + "type": "object", + }, + "schema_version": { + "enum": [ + "scope/taxonomy-evaluation/v1", + ], + "type": "string", + }, + "status": { + "enum": [ + "failed", + ], + "type": "string", + }, + "taxonomy": { + "additionalProperties": false, + "properties": { + "digest": { + "pattern": "^[a-f0-9]{64}$", + "type": "string", + }, + "id": { + "minLength": 1, + "type": "string", + }, + "version": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "id", + "version", + "digest", + ], + "type": "object", + }, + }, + "required": [ + "schema_version", + "taxonomy", + "element", + "implementation", + "input", + "status", + "error", + ], + "type": "object", + }, + ], + }, + "type": "array", + }, + "run": { + "additionalProperties": false, + "properties": { + "id": { + "minLength": 1, + "type": "string", + }, + "profile": { + "additionalProperties": false, + "properties": { + "id": { + "minLength": 1, + "type": "string", + }, + "label": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "id", + "label", + ], + "type": "object", + }, + "scenario": { + "additionalProperties": false, + "properties": { + "id": { + "minLength": 1, + "type": "string", + }, + "label": { + "minLength": 1, + "type": "string", + }, + "task": { + "type": "string", + }, + }, + "required": [ + "id", + "label", + ], + "type": "object", + }, + }, + "required": [ + "id", + "scenario", + "profile", + ], + "type": "object", + }, + }, + "required": [ + "run", + "results", + ], + "type": "object", + }, + "TokenUsage": { + "properties": { + "completionTokens": { + "type": "number", + }, + "promptTokens": { + "type": "number", + }, + "totalTokens": { + "type": "number", + }, + }, + "required": [ + "promptTokens", + "completionTokens", + "totalTokens", + ], + "type": "object", + }, + "TriggerReportsInput": { + "properties": { + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + }, + "required": [ + "requestId", + ], + "type": "object", + }, + "UpdateAgentInput": { + "properties": { + "available": { + "type": "boolean", + }, + "capabilities": { + "$ref": "#/components/schemas/AgentCapabilities", + }, + "defaultModel": { + "type": "string", + }, + "description": { + "type": "string", + }, + "modelProvider": { + "type": "string", + }, + "name": { + "type": "string", + }, + "supportedModels": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "type": "object", + }, + "UpdateCodebaseInput": { + "properties": { + "defaultBranch": { + "type": "string", + }, + "description": { + "type": "string", + }, + "name": { + "minLength": 1, + "type": "string", + }, + }, + "type": "object", + }, + "UpdateCriteriaInput": { + "properties": { + "dependsOn": { + "items": { + "type": "string", + }, + "type": "array", + }, + "gates": { + "items": { + "enum": [ + "select", + "build", + "test", + "run", + "deploy", + ], + "type": "string", + }, + "type": "array", + }, + "kind": { + "enum": [ + "gate", + "observation", + ], + "type": "string", + }, + "prompt": { + "type": "string", + }, + "subject": { + "enum": [ + "run", + "iteration", + ], + "type": "string", + }, + "taxonomyElementId": { + "enum": [ + "dimension:idiomatic-use", + "dimension:dependency-currency", + "dimension:configuration-correctness", + ], + "type": "string", + }, + }, + "type": "object", + }, + "UpdateExtensionInput": { + "properties": { + "description": { + "type": "string", + }, + "name": { + "type": "string", + }, + }, + "type": "object", + }, + "UpdateFeatureFlagInput": { + "properties": { + "enabled": { + "type": "boolean", + }, + }, + "required": [ + "enabled", + ], + "type": "object", + }, + "UpdateInsightInput": { + "properties": { + "category": { + "type": "string", + }, + "description": { + "type": "string", + }, + "tags": { + "items": { + "type": "string", + }, + "type": "array", + }, + "title": { + "type": "string", + }, + }, + "type": "object", + }, + "UpdateMcpServerInput": { + "properties": { + "args": { + "items": { + "type": "string", + }, + "type": "array", + }, + "command": { + "type": "string", + }, + "description": { + "type": "string", + }, + "env": { + "additionalProperties": { + "type": "string", + }, + "type": "object", + }, + "headers": { + "items": { + "$ref": "#/components/schemas/McpServerHeader", + }, + "type": "array", + }, + "name": { + "type": "string", + }, + "sessionMode": { + "enum": [ + "stateful", + "stateless", + ], + "type": "string", + }, + "type": { + "enum": [ + "sse", + "http", + "stdio", + ], + "type": "string", + }, + "url": { + "type": "string", + }, + "version": { + "type": "string", + }, + }, + "type": "object", + }, + "UpdateProfileIdentity": { + "properties": { + "description": { + "maxLength": 512, + "type": "string", + }, + "name": { + "maxLength": 128, + "minLength": 1, + "type": "string", + }, + }, + "type": "object", + }, + "UpdateProjectInput": { + "properties": { + "description": { + "type": "string", + }, + "name": { + "minLength": 1, + "type": "string", + }, + }, + "type": "object", + }, + "UpdatePromptFeatureInput": { + "properties": { + "prompt": { + "type": "string", + }, + }, + "type": "object", + }, + "UpdateReportTemplateInput": { + "properties": { + "description": { + "type": "string", + }, + "enabled": { + "type": "boolean", + }, + "model": { + "type": [ + "string", + "null", + ], + }, + "name": { + "type": "string", + }, + "systemPrompt": { + "allOf": [ + { + "$ref": "#/components/schemas/ReportTemplateSystemPrompt", + }, + { + "type": [ + "object", + "null", + ], + }, + ], + }, + "timeoutMs": { + "exclusiveMinimum": 0, + "type": [ + "integer", + "null", + ], + }, + "trigger": { + "$ref": "#/components/schemas/ReportTrigger", + }, + "userPrompt": { + "type": "string", + }, + }, + "type": "object", + }, + "UpdateResourceInput": { + "properties": { + "description": { + "type": "string", + }, + "name": { + "minLength": 1, + "type": "string", + }, + }, + "type": "object", + }, + "UpdateTaxonomyMetadataInput": { + "additionalProperties": false, + "properties": { + "description": { + "maxLength": 4096, + "type": "string", + }, + "name": { + "maxLength": 256, + "minLength": 1, + "type": "string", + }, + }, + "type": "object", + }, + "UserMeResponse": { + "properties": { + "displayName": { + "type": "string", + }, + "email": { + "type": "string", + }, + "id": { + "type": "string", + }, + "idp": { + "type": "string", + }, + "idpTenant": { + "type": "string", + }, + "role": { + "type": "string", + }, + }, + "required": [ + "id", + ], + "type": "object", + }, + "ValidateKeyInput": { + "properties": { + "token": { + "type": "string", + }, + }, + "required": [ + "token", + ], + "type": "object", + }, + }, + "securitySchemes": { + "bearerAuth": { + "bearerFormat": "JWT", + "description": "Use the unchanged IdP access token, without the Bearer prefix.", + "scheme": "bearer", + "type": "http", + }, + }, + }, + "info": { + "description": "REST API for the Scope platform — benchmarking AI coding agents", + "title": "Scope API", + "version": "1.0.0", + }, + "openapi": "3.1.0", + "paths": { + "/about": { + "get": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "buildTime": { + "type": "string", + }, + "description": { + "type": "string", + }, + "environment": { + "type": "string", + }, + "name": { + "type": "string", + }, + "version": { + "type": "string", + }, + "workers": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "required": [ + "name", + "version", + "buildTime", + "environment", + "description", + "workers", + ], + "type": "object", }, }, }, @@ -7986,28 +9980,174 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "404": { "description": "Experiment not found", }, - "409": { - "description": "Conflicting lifecycle/publication intent or invalid recoverability", + "409": { + "description": "Conflicting lifecycle/publication intent or invalid recoverability", + }, + }, + "summary": "Start or resume cascading Experiment restore", + "tags": [ + "Experiments", + ], + }, + }, + "/api/v1/experiments/{id}/retry": { + "post": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "minLength": 1, + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "experimentExecutionId": { + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "experimentExecutionId", + ], + "type": "object", + }, + }, + }, + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RetryExperimentResponse", + }, + }, + }, + "description": "Success", + }, + "404": { + "description": "Experiment not found", + }, + "409": { + "description": "Execution is stale or cannot be retried", + }, + }, + "summary": "Retry current experiment processing", + "tags": [ + "Experiments", + ], + }, + }, + "/api/v1/experiments/{id}/taxonomy-readout": { + "get": { + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TaxonomyReadoutReference", + }, + }, + }, + "description": "Success", + }, + }, + "summary": "Get typed Taxonome readout status", + "tags": [ + "Experiments", + ], + }, + }, + "/api/v1/experiments/{id}/taxonomy-readout/json": { + "get": { + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Success", + }, + }, + "summary": "Download Taxonome readout json", + "tags": [ + "Experiments", + ], + }, + }, + "/api/v1/experiments/{id}/taxonomy-readout/markdown": { + "get": { + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Success", }, }, - "summary": "Start or resume cascading Experiment restore", + "summary": "Download Taxonome readout markdown", "tags": [ "Experiments", ], }, }, - "/api/v1/experiments/{id}/retry": { + "/api/v1/experiments/{id}/taxonomy-readout/retry": { "post": { "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "minLength": 1, - "type": "string", - }, - }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -8021,43 +10161,19 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "experimentExecutionId": { - "minLength": 1, - "type": "string", - }, - }, - "required": [ - "experimentExecutionId", - ], - "type": "object", - }, - }, - }, - }, "responses": { - "201": { + "202": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RetryExperimentResponse", + "$ref": "#/components/schemas/TaxonomyReadoutReference", }, }, }, "description": "Success", }, - "404": { - "description": "Experiment not found", - }, - "409": { - "description": "Execution is stale or cannot be retried", - }, }, - "summary": "Retry current experiment processing", + "summary": "Generate or retry the typed Taxonome readout", "tags": [ "Experiments", ], @@ -11756,167 +13872,560 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, - { - "in": "query", - "name": "createdAfter", - "required": false, - "schema": { - "format": "date-time", - "type": [ - "string", - "null", - ], + { + "in": "query", + "name": "createdAfter", + "required": false, + "schema": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + { + "in": "query", + "name": "createdBefore", + "required": false, + "schema": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + }, + { + "in": "query", + "name": "turns", + "required": false, + "schema": { + "minimum": 0, + "type": [ + "integer", + "null", + ], + }, + }, + { + "in": "query", + "name": "turnsOp", + "required": false, + "schema": { + "enum": [ + "eq", + "gte", + "lte", + ], + "type": "string", + }, + }, + { + "in": "query", + "name": "maxIterations", + "required": false, + "schema": { + "minimum": 0, + "type": [ + "integer", + "null", + ], + }, + }, + { + "in": "query", + "name": "maxIterationsOp", + "required": false, + "schema": { + "enum": [ + "eq", + "gte", + "lte", + ], + "type": "string", + }, + }, + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/PaginatedRunsResponse", + }, + { + "$ref": "#/components/schemas/PaginatedRunGroupsResponse", + }, + ], + }, + }, + }, + "description": "Success", + }, + }, + "summary": "List requests", + "tags": [ + "Requests", + ], + }, + "post": { + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/CreateRequestInput", + }, + { + "default": undefined, + "properties": { + "agentVersion": { + "type": "string", + }, + "codebase": { + "type": "string", + }, + "count": { + "default": 1, + "deprecated": true, + "description": "Deprecated for composition. Use POST /api/v1/experiments and its requests array.", + "maximum": 10, + "minimum": 1, + "type": "number", + }, + "skills": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "type": "object", + }, + ], + }, + }, + }, + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/RequestResponse", + }, + { + "items": { + "$ref": "#/components/schemas/RequestResponse", + }, + "type": "array", + }, + ], + }, + }, + }, + "description": "Success", + }, + }, + "summary": "Submit request(s)", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/archive": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "ids": { + "items": { + "type": "string", + }, + "maxItems": 100, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "ids", + ], + "type": "object", + }, + }, + }, + }, + "responses": { + "201": { + "description": "Gzipped batch archive containing individual run archives", + }, + "400": { + "description": "Invalid input", + }, + "404": { + "description": "One or more runs not found", + }, + }, + "summary": "Download batch archive of multiple runs", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/bulk": { + "delete": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "ids": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "required": [ + "ids", + ], + "type": "object", + }, }, }, - { - "in": "query", - "name": "createdBefore", - "required": false, - "schema": { - "format": "date-time", - "type": [ - "string", - "null", - ], + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "deleted": { + "type": "number", + }, + "membershipRevision": { + "properties": { + "experimentId": { + "type": "string", + }, + "experimentRevisionId": { + "type": "string", + }, + "revisionNumber": { + "type": "integer", + }, + }, + "required": [ + "experimentId", + "experimentRevisionId", + "revisionNumber", + ], + "type": "object", + }, + "notFound": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "required": [ + "deleted", + "notFound", + ], + "type": "object", + }, + }, }, + "description": "Success", }, - { - "in": "query", - "name": "turns", - "required": false, - "schema": { - "minimum": 0, - "type": [ - "integer", - "null", - ], + "409": { + "description": "Nonterminal coding/post-processing/report work, deletion spans Experiments, or concurrent mutation", + }, + }, + "summary": "Bulk soft-delete requests", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/bulk-cancel": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "ids": { + "items": { + "type": "string", + }, + "maxItems": 100, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "ids", + ], + "type": "object", + }, }, }, - { - "in": "query", - "name": "turnsOp", - "required": false, - "schema": { - "enum": [ - "eq", - "gte", - "lte", - ], - "type": "string", + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "properties": { + "cancelled": { + "type": "number", + }, + "results": { + "items": { + "properties": { + "cancelled": { + "type": "boolean", + }, + "error": { + "type": "string", + }, + "id": { + "type": "string", + }, + "previousStatus": { + "type": "string", + }, + }, + "required": [ + "id", + "cancelled", + ], + "type": "object", + }, + "type": "array", + }, + "skipped": { + "type": "number", + }, + }, + "required": [ + "cancelled", + "skipped", + "results", + ], + "type": "object", + }, + }, }, + "description": "Success", }, - { - "in": "query", - "name": "maxIterations", - "required": false, - "schema": { - "minimum": 0, - "type": [ - "integer", - "null", - ], + }, + "summary": "Cancel multiple requests", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/bulk-pause": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "ids": { + "items": { + "type": "string", + }, + "maxItems": 100, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "ids", + ], + "type": "object", + }, }, }, - { - "in": "query", - "name": "maxIterationsOp", - "required": false, - "schema": { - "enum": [ - "eq", - "gte", - "lte", - ], - "type": "string", + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "properties": { + "skipped": { + "type": "number", + }, + "updated": { + "type": "number", + }, + }, + "required": [ + "updated", + "skipped", + ], + "type": "object", + }, + }, }, + "description": "Success", }, - { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "in": "query", - "name": "projectId", - "required": true, - "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", - "minLength": 1, - "type": "string", + }, + "summary": "Pause multiple requests", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/bulk-priority": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "ids": { + "items": { + "type": "string", + }, + "maxItems": 100, + "minItems": 1, + "type": "array", + }, + "priority": { + "type": "integer", + }, + }, + "required": [ + "ids", + "priority", + ], + "type": "object", + }, }, }, - ], + }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/PaginatedRunsResponse", + "properties": { + "skipped": { + "type": "number", }, - { - "$ref": "#/components/schemas/PaginatedRunGroupsResponse", + "updated": { + "type": "number", }, + }, + "required": [ + "updated", + "skipped", ], + "type": "object", }, }, }, "description": "Success", }, }, - "summary": "List requests", + "summary": "Set priority on multiple requests", "tags": [ "Requests", ], }, + }, + "/api/v1/requests/bulk-resubmit": { "post": { - "parameters": [ - { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "in": "query", - "name": "projectId", - "required": true, - "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", - "minLength": 1, - "type": "string", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BulkResubmitInput", + }, + }, + }, + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/RequestResponse", + }, + "type": "array", + }, + }, }, + "description": "Success", }, + }, + "summary": "Bulk resubmit requests", + "tags": [ + "Requests", ], + }, + }, + "/api/v1/requests/bulk-resume": { + "post": { "requestBody": { "content": { "application/json": { "schema": { - "allOf": [ - { - "$ref": "#/components/schemas/CreateRequestInput", - }, - { - "default": undefined, - "properties": { - "agentVersion": { - "type": "string", - }, - "codebase": { - "type": "string", - }, - "count": { - "default": 1, - "deprecated": true, - "description": "Deprecated for composition. Use POST /api/v1/experiments and its requests array.", - "maximum": 10, - "minimum": 1, - "type": "number", - }, - "skills": { - "items": { - "type": "string", - }, - "type": "array", - }, + "properties": { + "ids": { + "items": { + "type": "string", }, - "type": "object", + "maxItems": 100, + "minItems": 1, + "type": "array", }, + }, + "required": [ + "ids", ], + "type": "object", }, }, }, @@ -11926,41 +14435,45 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/RequestResponse", + "properties": { + "skipped": { + "type": "number", }, - { - "items": { - "$ref": "#/components/schemas/RequestResponse", - }, - "type": "array", + "updated": { + "type": "number", }, + }, + "required": [ + "updated", + "skipped", ], + "type": "object", }, }, }, "description": "Success", }, }, - "summary": "Submit request(s)", + "summary": "Resume multiple paused requests", "tags": [ "Requests", ], }, }, - "/api/v1/requests/archive": { + "/api/v1/requests/bulk-retry": { "post": { "requestBody": { "content": { "application/json": { "schema": { "properties": { + "force": { + "type": "boolean", + }, "ids": { "items": { "type": "string", }, - "maxItems": 100, "minItems": 1, "type": "array", }, @@ -11972,46 +14485,108 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, }, - }, + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "properties": { + "results": { + "items": { + "properties": { + "attemptNumber": { + "type": "integer", + }, + "error": { + "type": "string", + }, + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + }, + "required": [ + "requestId", + ], + "type": "object", + }, + "type": "array", + }, + "retried": { + "type": "integer", + }, + "skipped": { + "type": "integer", + }, + }, + "required": [ + "retried", + "skipped", + "results", + ], + "type": "object", + }, + }, + }, + "description": "Success", + }, + }, + "summary": "Bulk retry requests (start new attempts)", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/facets": { + "get": { + "description": "Returns every distinct value and its full-dataset count per categorical filter dimension for the Runs list rail. Counts are absolute over all non-deleted runs **in the given project**: they intentionally ignore the active search, date range, iteration, and categorical selections so every selectable value stays visible with a stable count. Requires ?projectId=. Computed with parallel $group aggregations (Cosmos has no $facet) and cached per-project for a short TTL.", + "parameters": [ + { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", + "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + ], "responses": { - "201": { - "description": "Gzipped batch archive containing individual run archives", - }, - "400": { - "description": "Invalid input", - }, - "404": { - "description": "One or more runs not found", + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RunFacetsResponse", + }, + }, + }, + "description": "Success", }, }, - "summary": "Download batch archive of multiple runs", + "summary": "List run filter facets", "tags": [ "Requests", ], }, }, - "/api/v1/requests/bulk": { + "/api/v1/requests/{id}": { "delete": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "ids": { - "items": { - "type": "string", - }, - "type": "array", - }, - }, - "required": [ - "ids", - ], - "type": "object", - }, + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", }, }, - }, + ], "responses": { "200": { "content": { @@ -12019,7 +14594,13 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "schema": { "properties": { "deleted": { - "type": "number", + "enum": [ + true, + ], + "type": "boolean", + }, + "id": { + "type": "string", }, "membershipRevision": { "properties": { @@ -12040,16 +14621,10 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "type": "object", }, - "notFound": { - "items": { - "type": "string", - }, - "type": "array", - }, }, "required": [ + "id", "deleted", - "notFound", ], "type": "object", }, @@ -12057,135 +14632,228 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "description": "Success", }, + "404": { + "description": "Not found", + }, "409": { - "description": "Nonterminal coding/post-processing/report work, deletion spans Experiments, or concurrent mutation", + "description": "Nonterminal coding/post-processing/report work or concurrent mutation", + }, + "410": { + "description": "Already deleted", }, }, - "summary": "Bulk soft-delete requests", + "summary": "Soft-delete request", "tags": [ "Requests", ], }, - }, - "/api/v1/requests/bulk-cancel": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "ids": { - "items": { - "type": "string", - }, - "maxItems": 100, - "minItems": 1, - "type": "array", - }, + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RequestResponse", }, - "required": [ - "ids", - ], - "type": "object", }, }, + "description": "Success", + }, + "404": { + "description": "Not found", + }, + }, + "summary": "Get request", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/{id}/archive": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Gzipped run archive", + }, + "404": { + "description": "Not found", + }, + }, + "summary": "Download full run archive", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/{id}/atif": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + { + "description": "The iteration number (1-based)", + "in": "query", + "name": "iteration", + "required": true, + "schema": { + "description": "The iteration number (1-based)", + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "ATIF v1.7 trajectory JSON file", + }, + "400": { + "description": "Missing or invalid iteration", + }, + "404": { + "description": "Not found", }, }, + "summary": "Download ATIF trajectory file for a specific iteration", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/{id}/cancel": { + "post": { "responses": { "201": { "content": { "application/json": { "schema": { "properties": { - "cancelled": { - "type": "number", + "id": { + "type": "string", }, - "results": { - "items": { - "properties": { - "cancelled": { - "type": "boolean", - }, - "error": { - "type": "string", - }, - "id": { - "type": "string", - }, - "previousStatus": { - "type": "string", - }, - }, - "required": [ - "id", - "cancelled", - ], - "type": "object", - }, - "type": "array", + "outcome": { + "type": "string", }, - "skipped": { - "type": "number", + "previousStatus": { + "type": "string", + }, + "status": { + "type": "string", }, }, "required": [ - "cancelled", - "skipped", - "results", + "id", + "previousStatus", + "status", + "outcome", ], "type": "object", }, }, }, - "description": "Success", + "description": "Success", + }, + }, + "summary": "Cancel a request (marks as done/failed and signals worker to exit)", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/{id}/har": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "HAR-format JSON file", + }, + "404": { + "description": "Not found", }, }, - "summary": "Cancel multiple requests", + "summary": "Download HAR file", "tags": [ "Requests", ], }, }, - "/api/v1/requests/bulk-pause": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "ids": { - "items": { - "type": "string", - }, - "maxItems": 100, - "minItems": 1, - "type": "array", - }, - }, - "required": [ - "ids", - ], - "type": "object", - }, + "/api/v1/requests/{id}/logs": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", }, }, + ], + "responses": { + "200": { + "description": "Server-sent event stream of log entries", + }, + "404": { + "description": "Not found", + }, }, + "summary": "Stream request logs (SSE)", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/{id}/pause": { + "post": { "responses": { "201": { "content": { "application/json": { "schema": { "properties": { - "skipped": { - "type": "number", + "id": { + "type": "string", }, - "updated": { - "type": "number", + "status": { + "type": "string", }, }, "required": [ - "updated", - "skipped", + "id", + "status", ], "type": "object", }, @@ -12194,33 +14862,24 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "description": "Success", }, }, - "summary": "Pause multiple requests", + "summary": "Pause a single request", "tags": [ "Requests", ], }, }, - "/api/v1/requests/bulk-priority": { + "/api/v1/requests/{id}/priority": { "post": { "requestBody": { "content": { "application/json": { "schema": { "properties": { - "ids": { - "items": { - "type": "string", - }, - "maxItems": 100, - "minItems": 1, - "type": "array", - }, "priority": { "type": "integer", }, }, "required": [ - "ids", "priority", ], "type": "object", @@ -12234,16 +14893,16 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "application/json": { "schema": { "properties": { - "skipped": { - "type": "number", + "id": { + "type": "string", }, - "updated": { + "priority": { "type": "number", }, }, "required": [ - "updated", - "skipped", + "id", + "priority", ], "type": "object", }, @@ -12252,30 +14911,31 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "description": "Success", }, }, - "summary": "Set priority on multiple requests", + "summary": "Set priority on a single request", "tags": [ "Requests", ], }, }, - "/api/v1/requests/bulk-resubmit": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BulkResubmitInput", - }, + "/api/v1/requests/{id}/reports": { + "get": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", }, }, - }, + ], "responses": { - "201": { + "200": { "content": { "application/json": { "schema": { "items": { - "$ref": "#/components/schemas/RequestResponse", + "$ref": "#/components/schemas/ReportResponse", }, "type": "array", }, @@ -12283,53 +14943,34 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "description": "Success", }, + "404": { + "description": "Run not found", + }, }, - "summary": "Bulk resubmit requests", + "summary": "Get reports for request", "tags": [ - "Requests", + "Reports", ], }, }, - "/api/v1/requests/bulk-resume": { + "/api/v1/requests/{id}/resume": { "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "ids": { - "items": { - "type": "string", - }, - "maxItems": 100, - "minItems": 1, - "type": "array", - }, - }, - "required": [ - "ids", - ], - "type": "object", - }, - }, - }, - }, "responses": { "201": { "content": { "application/json": { "schema": { "properties": { - "skipped": { - "type": "number", + "id": { + "type": "string", }, - "updated": { - "type": "number", + "status": { + "type": "string", }, }, "required": [ - "updated", - "skipped", + "id", + "status", ], "type": "object", }, @@ -12338,14 +14979,24 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "description": "Success", }, }, - "summary": "Resume multiple paused requests", + "summary": "Resume a paused request", "tags": [ "Requests", ], }, }, - "/api/v1/requests/bulk-retry": { + "/api/v1/requests/{id}/retry": { "post": { + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + ], "requestBody": { "content": { "application/json": { @@ -12354,17 +15005,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "force": { "type": "boolean", }, - "ids": { - "items": { - "type": "string", - }, - "minItems": 1, - "type": "array", - }, }, - "required": [ - "ids", - ], "type": "object", }, }, @@ -12376,91 +15017,45 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "application/json": { "schema": { "properties": { - "results": { - "items": { - "properties": { - "attemptNumber": { - "type": "integer", - }, - "error": { - "type": "string", - }, - "requestId": { - "type": "string", - }, - "runId": { - "type": "string", - }, - }, - "required": [ - "requestId", - ], - "type": "object", - }, - "type": "array", - }, - "retried": { - "type": "integer", - }, - "skipped": { + "attemptNumber": { "type": "integer", }, - }, - "required": [ - "retried", - "skipped", - "results", - ], - "type": "object", - }, - }, - }, - "description": "Success", - }, - }, - "summary": "Bulk retry requests (start new attempts)", - "tags": [ - "Requests", - ], - }, - }, - "/api/v1/requests/facets": { - "get": { - "description": "Returns every distinct value and its full-dataset count per categorical filter dimension for the Runs list rail. Counts are absolute over all non-deleted runs **in the given project**: they intentionally ignore the active search, date range, iteration, and categorical selections so every selectable value stays visible with a stable count. Requires ?projectId=. Computed with parallel $group aggregations (Cosmos has no $facet) and cached per-project for a short TTL.", - "parameters": [ - { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "in": "query", - "name": "projectId", - "required": true, - "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", - "minLength": 1, - "type": "string", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RunFacetsResponse", + "requestId": { + "type": "string", + }, + "runId": { + "type": "string", + }, + }, + "required": [ + "requestId", + "runId", + "attemptNumber", + ], + "type": "object", }, }, }, "description": "Success", }, + "404": { + "description": "Request not found", + }, + "409": { + "description": "Conflict — request is not in a retryable state, or a concurrent retry won the race", + }, + "422": { + "description": "Cannot retry — current run not yet terminal", + }, }, - "summary": "List run filter facets", + "summary": "Retry a request (start a new attempt)", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}": { - "delete": { + "/api/v1/requests/{id}/runs": { + "get": { "parameters": [ { "in": "path", @@ -12476,61 +15071,26 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "properties": { - "deleted": { - "enum": [ - true, - ], - "type": "boolean", - }, - "id": { - "type": "string", - }, - "membershipRevision": { - "properties": { - "experimentId": { - "type": "string", - }, - "experimentRevisionId": { - "type": "string", - }, - "revisionNumber": { - "type": "integer", - }, - }, - "required": [ - "experimentId", - "experimentRevisionId", - "revisionNumber", - ], - "type": "object", - }, + "items": { + "$ref": "#/components/schemas/RunState", }, - "required": [ - "id", - "deleted", - ], - "type": "object", + "type": "array", }, }, }, "description": "Success", }, "404": { - "description": "Not found", - }, - "409": { - "description": "Nonterminal coding/post-processing/report work or concurrent mutation", - }, - "410": { - "description": "Already deleted", + "description": "Request not found", }, }, - "summary": "Soft-delete request", + "summary": "List attempts for a request", "tags": [ "Requests", ], }, + }, + "/api/v1/requests/{id}/runs/{runId}": { "get": { "parameters": [ { @@ -12541,29 +15101,37 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "path", + "name": "runId", + "required": true, + "schema": { + "type": "string", + }, + }, ], "responses": { "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RequestResponse", + "$ref": "#/components/schemas/RunState", }, }, }, "description": "Success", }, "404": { - "description": "Not found", + "description": "Request or run not found", }, }, - "summary": "Get request", + "summary": "Get a single attempt", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/archive": { + "/api/v1/requests/{id}/runs/{runId}/archive": { "get": { "parameters": [ { @@ -12574,6 +15142,14 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "path", + "name": "runId", + "required": true, + "schema": { + "type": "string", + }, + }, ], "responses": { "200": { @@ -12583,13 +15159,13 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "description": "Not found", }, }, - "summary": "Download full run archive", + "summary": "Download full run archive for a specific attempt", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/atif": { + "/api/v1/requests/{id}/runs/{runId}/atif": { "get": { "parameters": [ { @@ -12600,6 +15176,14 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "path", + "name": "runId", + "required": true, + "schema": { + "type": "string", + }, + }, { "description": "The iteration number (1-based)", "in": "query", @@ -12622,53 +15206,13 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "description": "Not found", }, }, - "summary": "Download ATIF trajectory file for a specific iteration", - "tags": [ - "Requests", - ], - }, - }, - "/api/v1/requests/{id}/cancel": { - "post": { - "responses": { - "201": { - "content": { - "application/json": { - "schema": { - "properties": { - "id": { - "type": "string", - }, - "outcome": { - "type": "string", - }, - "previousStatus": { - "type": "string", - }, - "status": { - "type": "string", - }, - }, - "required": [ - "id", - "previousStatus", - "status", - "outcome", - ], - "type": "object", - }, - }, - }, - "description": "Success", - }, - }, - "summary": "Cancel a request (marks as done/failed and signals worker to exit)", + "summary": "Download ATIF trajectory file for a specific attempt and iteration", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/har": { + "/api/v1/requests/{id}/runs/{runId}/har": { "get": { "parameters": [ { @@ -12679,6 +15223,14 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "path", + "name": "runId", + "required": true, + "schema": { + "type": "string", + }, + }, ], "responses": { "200": { @@ -12688,13 +15240,13 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "description": "Not found", }, }, - "summary": "Download HAR file", + "summary": "Download HAR file for a specific attempt", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/logs": { + "/api/v1/requests/{id}/runs/{runId}/logs": { "get": { "parameters": [ { @@ -12704,104 +15256,31 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "schema": { "type": "string", }, - }, - ], - "responses": { - "200": { - "description": "Server-sent event stream of log entries", - }, - "404": { - "description": "Not found", - }, - }, - "summary": "Stream request logs (SSE)", - "tags": [ - "Requests", - ], - }, - }, - "/api/v1/requests/{id}/pause": { - "post": { - "responses": { - "201": { - "content": { - "application/json": { - "schema": { - "properties": { - "id": { - "type": "string", - }, - "status": { - "type": "string", - }, - }, - "required": [ - "id", - "status", - ], - "type": "object", - }, - }, - }, - "description": "Success", - }, - }, - "summary": "Pause a single request", - "tags": [ - "Requests", - ], - }, - }, - "/api/v1/requests/{id}/priority": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "priority": { - "type": "integer", - }, - }, - "required": [ - "priority", - ], - "type": "object", - }, - }, - }, - }, - "responses": { - "201": { - "content": { - "application/json": { - "schema": { - "properties": { - "id": { - "type": "string", - }, - "priority": { - "type": "number", - }, - }, - "required": [ - "id", - "priority", - ], - "type": "object", - }, - }, - }, - "description": "Success", + }, + { + "in": "path", + "name": "runId", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Server-sent event stream of log entries", + }, + "404": { + "description": "Not found", }, }, - "summary": "Set priority on a single request", + "summary": "Stream logs for a specific attempt (SSE)", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/reports": { + "/api/v1/requests/{id}/runs/{runId}/snapshots/{iteration}": { "get": { "parameters": [ { @@ -12812,65 +15291,92 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "path", + "name": "runId", + "required": true, + "schema": { + "type": "string", + }, + }, + { + "in": "path", + "name": "iteration", + "required": true, + "schema": { + "type": "string", + }, + }, + ], + "responses": { + "200": { + "description": "Gzipped snapshot archive", + }, + "404": { + "description": "Not found", + }, + }, + "summary": "Download iteration snapshot for a specific attempt", + "tags": [ + "Requests", ], + }, + }, + "/api/v1/requests/{id}/runs/{runId}/taxonomy-evaluation": { + "get": { "responses": { "200": { "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/ReportResponse", - }, - "type": "array", + "$ref": "#/components/schemas/TaxonomyEvaluationReference", }, }, }, "description": "Success", }, - "404": { - "description": "Run not found", + }, + "summary": "Get the exact-run Taxonome result reference", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/{id}/runs/{runId}/taxonomy-evaluation/content": { + "get": { + "responses": { + "200": { + "description": "Success", }, }, - "summary": "Get reports for request", + "summary": "Download the exact-run compressed Taxonome result", "tags": [ - "Reports", + "Requests", ], }, }, - "/api/v1/requests/{id}/resume": { - "post": { + "/api/v1/requests/{id}/runs/{runId}/taxonomy-evaluation/json": { + "get": { "responses": { - "201": { + "200": { "content": { "application/json": { "schema": { - "properties": { - "id": { - "type": "string", - }, - "status": { - "type": "string", - }, - }, - "required": [ - "id", - "status", - ], - "type": "object", + "$ref": "#/components/schemas/TaxonomyRunResultResponse", }, }, }, "description": "Success", }, }, - "summary": "Resume a paused request", + "summary": "Get the validated exact-run Taxonome result JSON", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/retry": { - "post": { + "/api/v1/requests/{id}/runs/{runId}/tool-calls": { + "get": { "parameters": [ { "in": "path", @@ -12880,65 +15386,30 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "force": { - "type": "boolean", - }, - }, - "type": "object", - }, + { + "in": "path", + "name": "runId", + "required": true, + "schema": { + "type": "string", }, }, - }, + ], "responses": { - "201": { - "content": { - "application/json": { - "schema": { - "properties": { - "attemptNumber": { - "type": "integer", - }, - "requestId": { - "type": "string", - }, - "runId": { - "type": "string", - }, - }, - "required": [ - "requestId", - "runId", - "attemptNumber", - ], - "type": "object", - }, - }, - }, - "description": "Success", + "200": { + "description": "JSONL stream — one ToolCall per line", }, "404": { - "description": "Request not found", - }, - "409": { - "description": "Conflict — request is not in a retryable state, or a concurrent retry won the race", - }, - "422": { - "description": "Cannot retry — current run not yet terminal", + "description": "Not found", }, }, - "summary": "Retry a request (start a new attempt)", + "summary": "Download per-iteration tool-calls JSONL for a specific attempt", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/runs": { + "/api/v1/requests/{id}/runs/{runId}/video": { "get": { "parameters": [ { @@ -12949,32 +15420,30 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "path", + "name": "runId", + "required": true, + "schema": { + "type": "string", + }, + }, ], "responses": { "200": { - "content": { - "application/json": { - "schema": { - "items": { - "$ref": "#/components/schemas/RunState", - }, - "type": "array", - }, - }, - }, - "description": "Success", + "description": "WebM video recording (supports Range requests)", }, "404": { - "description": "Request not found", + "description": "Not found", }, }, - "summary": "List attempts for a request", + "summary": "Download session recording for a specific attempt", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/runs/{runId}": { + "/api/v1/requests/{id}/snapshots/{iteration}": { "get": { "parameters": [ { @@ -12987,7 +15456,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, { "in": "path", - "name": "runId", + "name": "iteration", "required": true, "schema": { "type": "string", @@ -12996,26 +15465,19 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "responses": { "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RunState", - }, - }, - }, - "description": "Success", + "description": "Gzipped snapshot archive", }, "404": { - "description": "Request or run not found", + "description": "Not found", }, }, - "summary": "Get a single attempt", + "summary": "Download iteration snapshot", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/runs/{runId}/archive": { + "/api/v1/requests/{id}/tool-calls": { "get": { "parameters": [ { @@ -13026,9 +15488,27 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + ], + "responses": { + "200": { + "description": "JSONL stream — one ToolCall per line", + }, + "404": { + "description": "Not found", + }, + }, + "summary": "Download per-iteration tool-calls JSONL", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/requests/{id}/video": { + "get": { + "parameters": [ { "in": "path", - "name": "runId", + "name": "id", "required": true, "schema": { "type": "string", @@ -13037,66 +15517,96 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "responses": { "200": { - "description": "Gzipped run archive", + "description": "WebM video recording (supports Range requests)", }, "404": { "description": "Not found", }, }, - "summary": "Download full run archive for a specific attempt", + "summary": "Download session recording", "tags": [ "Requests", ], }, }, - "/api/v1/requests/{id}/runs/{runId}/atif": { + "/api/v1/resources": { "get": { "parameters": [ { - "in": "path", - "name": "id", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, - { - "in": "path", - "name": "runId", - "required": true, - "schema": { - "type": "string", + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/ResourceResponse", + }, + "type": "array", + }, + }, }, + "description": "Success", }, + }, + "summary": "List all resources", + "tags": [ + "Resources", + ], + }, + "post": { + "parameters": [ { - "description": "The iteration number (1-based)", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", - "name": "iteration", + "name": "projectId", "required": true, "schema": { - "description": "The iteration number (1-based)", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateResourceInput", + }, + }, + }, + }, "responses": { - "200": { - "description": "ATIF v1.7 trajectory JSON file", + "201": { + "description": "Success", }, "400": { - "description": "Missing or invalid iteration", + "description": "Invalid input", }, - "404": { - "description": "Not found", + "409": { + "description": "Resource slug or revision ref already exists in this project", }, }, - "summary": "Download ATIF trajectory file for a specific attempt and iteration", + "summary": "Create a resource and its first revision", "tags": [ - "Requests", + "Resources", ], }, }, - "/api/v1/requests/{id}/runs/{runId}/har": { + "/api/v1/resources/revisions/{id}": { "get": { "parameters": [ { @@ -13108,30 +15618,41 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, { - "in": "path", - "name": "runId", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, ], "responses": { "200": { - "description": "HAR-format JSON file", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResourceRevisionResponse", + }, + }, + }, + "description": "Success", }, "404": { - "description": "Not found", + "description": "Revision not found", }, }, - "summary": "Download HAR file for a specific attempt", + "summary": "Get resource revision by id", "tags": [ - "Requests", + "Resource Revisions", ], }, }, - "/api/v1/requests/{id}/runs/{runId}/logs": { - "get": { + "/api/v1/resources/{id}": { + "delete": { "parameters": [ { "in": "path", @@ -13142,29 +15663,31 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, { - "in": "path", - "name": "runId", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, ], "responses": { - "200": { - "description": "Server-sent event stream of log entries", + "204": { + "description": "Success", }, "404": { - "description": "Not found", + "description": "Resource not found", }, }, - "summary": "Stream logs for a specific attempt (SSE)", + "summary": "Delete a resource", "tags": [ - "Requests", + "Resources", ], }, - }, - "/api/v1/requests/{id}/runs/{runId}/snapshots/{iteration}": { "get": { "parameters": [ { @@ -13176,38 +15699,39 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, { - "in": "path", - "name": "runId", - "required": true, - "schema": { - "type": "string", - }, - }, - { - "in": "path", - "name": "iteration", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, ], "responses": { "200": { - "description": "Gzipped snapshot archive", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResourceResponse", + }, + }, + }, + "description": "Success", }, "404": { - "description": "Not found", + "description": "Resource not found", }, }, - "summary": "Download iteration snapshot for a specific attempt", + "summary": "Get a resource", "tags": [ - "Requests", + "Resources", ], }, - }, - "/api/v1/requests/{id}/runs/{runId}/tool-calls": { - "get": { + "patch": { "parameters": [ { "in": "path", @@ -13218,29 +15742,49 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, { - "in": "path", - "name": "runId", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateResourceInput", + }, + }, + }, + }, "responses": { "200": { - "description": "JSONL stream — one ToolCall per line", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResourceResponse", + }, + }, + }, + "description": "Success", }, "404": { - "description": "Not found", + "description": "Resource not found", }, }, - "summary": "Download per-iteration tool-calls JSONL for a specific attempt", + "summary": "Update a resource", "tags": [ - "Requests", + "Resources", ], }, }, - "/api/v1/requests/{id}/runs/{runId}/video": { + "/api/v1/resources/{id}/revisions": { "get": { "parameters": [ { @@ -13252,9 +15796,21 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, { - "in": "path", - "name": "runId", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, + "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, + "type": "string", + }, + }, + { + "in": "query", + "name": "limit", + "required": false, "schema": { "type": "string", }, @@ -13262,20 +15818,28 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "responses": { "200": { - "description": "WebM video recording (supports Range requests)", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/ResourceRevisionResponse", + }, + "type": "array", + }, + }, + }, + "description": "Success", }, "404": { - "description": "Not found", + "description": "Resource not found", }, }, - "summary": "Download session recording for a specific attempt", + "summary": "List resource revisions", "tags": [ - "Requests", + "Resource Revisions", ], }, - }, - "/api/v1/requests/{id}/snapshots/{iteration}": { - "get": { + "post": { "parameters": [ { "in": "path", @@ -13286,29 +15850,45 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, { - "in": "path", - "name": "iteration", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateResourceRevisionInput", + }, + }, + }, + }, "responses": { - "200": { - "description": "Gzipped snapshot archive", + "201": { + "description": "Success", }, "404": { - "description": "Not found", + "description": "Resource not found", + }, + "409": { + "description": "Resource revision ref already exists in this project", }, }, - "summary": "Download iteration snapshot", + "summary": "Create a resource revision", "tags": [ - "Requests", + "Resource Revisions", ], }, }, - "/api/v1/requests/{id}/tool-calls": { + "/api/v1/resources/{id}/revisions/latest": { "get": { "parameters": [ { @@ -13319,50 +15899,59 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, - ], - "responses": { - "200": { - "description": "JSONL stream — one ToolCall per line", - }, - "404": { - "description": "Not found", - }, - }, - "summary": "Download per-iteration tool-calls JSONL", - "tags": [ - "Requests", - ], - }, - }, - "/api/v1/requests/{id}/video": { - "get": { - "parameters": [ { - "in": "path", - "name": "id", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, ], "responses": { "200": { - "description": "WebM video recording (supports Range requests)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ResourceRevisionResponse", + }, + }, + }, + "description": "Success", }, "404": { - "description": "Not found", + "description": "Revision not found", }, }, - "summary": "Download session recording", + "summary": "Get the latest resource revision", "tags": [ - "Requests", + "Resource Revisions", ], }, }, - "/api/v1/resources": { + "/api/v1/resources/{id}/revisions/{revisionNumber}": { "get": { "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string", + }, + }, + { + "in": "path", + "name": "revisionNumber", + "required": true, + "schema": { + "type": "string", + }, + }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -13381,21 +15970,23 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/ResourceResponse", - }, - "type": "array", + "$ref": "#/components/schemas/ResourceRevisionResponse", }, }, }, "description": "Success", }, + "404": { + "description": "Revision not found", + }, }, - "summary": "List all resources", + "summary": "Get a resource revision by resource and revision number", "tags": [ - "Resources", + "Resource Revisions", ], }, + }, + "/api/v1/runs/upload": { "post": { "parameters": [ { @@ -13411,43 +16002,50 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateResourceInput", - }, - }, - }, - }, "responses": { "201": { "description": "Success", }, - "400": { - "description": "Invalid input", - }, - "409": { - "description": "Resource slug or revision ref already exists in this project", - }, }, - "summary": "Create a resource and its first revision", + "summary": "Import run archive", "tags": [ - "Resources", + "Requests", ], }, }, - "/api/v1/resources/revisions/{id}": { - "get": { + "/api/v1/runs/upload-batch": { + "post": { "parameters": [ { - "in": "path", - "name": "id", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, + ], + "responses": { + "207": { + "description": "Multi-status: 201 if all runs imported, 400 if none imported, 207 if partial", + }, + "400": { + "description": "No archive uploaded, empty archive, or all runs failed", + }, + }, + "summary": "Import batch run archive", + "tags": [ + "Requests", + ], + }, + }, + "/api/v1/skill-revisions/by-ref/{ref}(*)": { + "get": { + "parameters": [ { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -13466,33 +16064,25 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ResourceRevisionResponse", + "$ref": "#/components/schemas/SkillRevisionResponse", }, }, }, "description": "Success", }, "404": { - "description": "Revision not found", + "description": "Skill revision not found", }, }, - "summary": "Get resource revision by id", + "summary": "Get skill revision by ref", "tags": [ - "Resource Revisions", + "Skill Revisions", ], }, }, - "/api/v1/resources/{id}": { - "delete": { + "/api/v1/skill-revisions/by-ref/{ref}(*)/archive": { + "get": { "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -13507,28 +16097,48 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, ], "responses": { - "204": { - "description": "Success", + "200": { + "description": "Binary tar.gz archive", }, "404": { - "description": "Resource not found", + "description": "Skill revision or archive not found", + }, + "500": { + "description": "Blob storage error", }, }, - "summary": "Delete a resource", + "summary": "Download skill revision archive", "tags": [ - "Resources", + "Skill Revisions", ], }, + }, + "/api/v1/skill-revisions/{id}": { "get": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillRevisionResponse", + }, + }, }, + "description": "Success", + }, + "404": { + "description": "Skill revision not found", }, + }, + "summary": "Get skill revision by ID", + "tags": [ + "Skill Revisions", + ], + }, + }, + "/api/v1/skills": { + "get": { + "parameters": [ { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -13547,31 +16157,23 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ResourceResponse", + "items": { + "$ref": "#/components/schemas/SkillResponse", + }, + "type": "array", }, }, }, "description": "Success", }, - "404": { - "description": "Resource not found", - }, }, - "summary": "Get a resource", + "summary": "List all skills", "tags": [ - "Resources", + "Skills", ], }, - "patch": { + "post": { "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -13589,38 +16191,35 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UpdateResourceInput", + "$ref": "#/components/schemas/CreateSkillInput", }, }, }, }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ResourceResponse", + "$ref": "#/components/schemas/SkillResponse", }, }, }, "description": "Success", }, - "404": { - "description": "Resource not found", - }, }, - "summary": "Update a resource", + "summary": "Create or import a skill", "tags": [ - "Resources", + "Skills", ], }, }, - "/api/v1/resources/{id}/revisions": { + "/api/v1/skills/discover": { "get": { "parameters": [ { - "in": "path", - "name": "id", + "in": "query", + "name": "source", "required": true, "schema": { "type": "string", @@ -13638,14 +16237,6 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, - { - "in": "query", - "name": "limit", - "required": false, - "schema": { - "type": "string", - }, - }, ], "responses": { "200": { @@ -13653,7 +16244,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "application/json": { "schema": { "items": { - "$ref": "#/components/schemas/ResourceRevisionResponse", + "$ref": "#/components/schemas/SkillDiscoveryResult", }, "type": "array", }, @@ -13661,25 +16252,41 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, "description": "Success", }, + "400": { + "description": "Missing or malformed source parameter", + }, "404": { - "description": "Resource not found", + "description": "Repository not found", + }, + "502": { + "description": "GitHub API error", }, }, - "summary": "List resource revisions", + "summary": "Discover skills in a GitHub repository", "tags": [ - "Resource Revisions", + "Skills", ], }, - "post": { + }, + "/api/v1/skills/search": { + "get": { "parameters": [ { - "in": "path", - "name": "id", + "in": "query", + "name": "q", "required": true, "schema": { "type": "string", }, }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "type": "string", + }, + }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -13693,51 +16300,56 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateResourceRevisionInput", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/SkillSearchResult", + }, + "type": "array", + }, }, }, - }, - }, - "responses": { - "201": { "description": "Success", }, - "404": { - "description": "Resource not found", - }, - "409": { - "description": "Resource revision ref already exists in this project", + "400": { + "description": "Missing query parameter", }, }, - "summary": "Create a resource revision", + "summary": "Search skills (internal + external)", "tags": [ - "Resource Revisions", + "Skills", ], }, }, - "/api/v1/resources/{id}/revisions/latest": { + "/api/v1/skills/search/external": { "get": { "parameters": [ { - "in": "path", - "name": "id", + "in": "query", + "name": "q", "required": true, "schema": { "type": "string", }, }, { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "limit", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "description": "Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project.", "in": "query", "name": "projectId", - "required": true, + "required": false, "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", + "description": "Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project.", "minLength": 1, "type": "string", }, @@ -13748,41 +16360,56 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ResourceRevisionResponse", + "items": { + "$ref": "#/components/schemas/SkillSearchResult", + }, + "type": "array", }, }, }, "description": "Success", }, - "404": { - "description": "Revision not found", + "400": { + "description": "Missing query parameter", }, }, - "summary": "Get the latest resource revision", + "summary": "Search external skills registry", "tags": [ - "Resource Revisions", + "Skills", ], }, }, - "/api/v1/resources/{id}/revisions/{revisionNumber}": { - "get": { + "/api/v1/skills/{id}(*)": { + "delete": { "parameters": [ { - "in": "path", - "name": "id", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, - { - "in": "path", - "name": "revisionNumber", - "required": true, - "schema": { - "type": "string", - }, + ], + "responses": { + "204": { + "description": "Success", }, + "404": { + "description": "Skill not found", + }, + }, + "summary": "Soft-delete a skill", + "tags": [ + "Skills", + ], + }, + "get": { + "parameters": [ { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -13801,23 +16428,23 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ResourceRevisionResponse", + "$ref": "#/components/schemas/SkillResponse", }, }, }, "description": "Success", }, "404": { - "description": "Revision not found", + "description": "Skill not found", }, }, - "summary": "Get a resource revision by resource and revision number", + "summary": "Get skill by slug", "tags": [ - "Resource Revisions", + "Skills", ], }, }, - "/api/v1/runs/upload": { + "/api/v1/skills/{id}(*)/resolve": { "post": { "parameters": [ { @@ -13835,18 +16462,39 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` ], "responses": { "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillRevisionResponse", + }, + }, + }, "description": "Success", }, + "404": { + "description": "Skill not found", + }, + "500": { + "description": "Blob storage error", + }, }, - "summary": "Import run archive", + "summary": "Trigger skill resolution", "tags": [ - "Requests", + "Skills", ], }, }, - "/api/v1/runs/upload-batch": { - "post": { + "/api/v1/skills/{id}(*)/revisions": { + "get": { "parameters": [ + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "type": "string", + }, + }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -13861,22 +16509,70 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, ], "responses": { - "207": { - "description": "Multi-status: 201 if all runs imported, 400 if none imported, 207 if partial", + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/SkillRevisionResponse", + }, + "type": "array", + }, + }, + }, + "description": "Success", }, - "400": { - "description": "No archive uploaded, empty archive, or all runs failed", + "404": { + "description": "Skill not found", }, }, - "summary": "Import batch run archive", + "summary": "List skill revisions", "tags": [ - "Requests", + "Skills", ], }, }, - "/api/v1/skill-revisions/by-ref/{ref}(*)": { + "/api/v1/task-prompts": { "get": { "parameters": [ + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "type": [ + "number", + "null", + ], + }, + }, + { + "in": "query", + "name": "offset", + "required": false, + "schema": { + "type": [ + "number", + "null", + ], + }, + }, + { + "in": "query", + "name": "search", + "required": false, + "schema": { + "type": "string", + }, + }, + { + "in": "query", + "name": "type", + "required": false, + "schema": { + "$ref": "#/components/schemas/PromptType", + }, + }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -13895,24 +16591,42 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SkillRevisionResponse", + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/TaskPromptResponse", + }, + "type": "array", + }, + "limit": { + "type": "number", + }, + "offset": { + "type": "number", + }, + "total": { + "type": "number", + }, + }, + "required": [ + "items", + "total", + "limit", + "offset", + ], + "type": "object", }, }, }, "description": "Success", }, - "404": { - "description": "Skill revision not found", - }, }, - "summary": "Get skill revision by ref", + "summary": "List task prompts", "tags": [ - "Skill Revisions", + "Task Prompts", ], }, - }, - "/api/v1/skill-revisions/by-ref/{ref}(*)/archive": { - "get": { + "post": { "parameters": [ { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", @@ -13927,58 +16641,95 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, ], - "responses": { - "200": { - "description": "Binary tar.gz archive", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateTaskPromptInput", + }, + }, }, - "404": { - "description": "Skill revision or archive not found", + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TaskPromptResponse", + }, + }, + }, + "description": "Success", }, - "500": { - "description": "Blob storage error", + "400": { + "description": "Empty text string", }, }, - "summary": "Download skill revision archive", + "summary": "Create or find task prompt", "tags": [ - "Skill Revisions", + "Task Prompts", ], }, }, - "/api/v1/skill-revisions/{id}": { - "get": { + "/api/v1/task-prompts/generate": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "description": { + "type": "string", + }, + "existingPrompt": { + "type": "string", + }, + }, + "type": "object", + }, + }, + }, + }, "responses": { "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SkillRevisionResponse", + "properties": { + "tasks": { + "items": { + "type": "string", + }, + "type": "array", + }, + }, + "required": [ + "tasks", + ], + "type": "object", }, }, }, "description": "Success", }, - "404": { - "description": "Skill revision not found", + "503": { + "description": "LLM not configured", }, }, - "summary": "Get skill revision by ID", + "summary": "Generate task prompts", "tags": [ - "Skill Revisions", + "Task Prompts", ], }, }, - "/api/v1/skills": { - "get": { + "/api/v1/task-prompts/{id}": { + "delete": { "parameters": [ { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "in": "query", - "name": "projectId", + "in": "path", + "name": "id", "required": true, "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", - "minLength": 1, "type": "string", }, }, @@ -13988,83 +16739,69 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/SkillResponse", + "properties": { + "deleted": { + "type": "boolean", + }, }, - "type": "array", + "required": [ + "deleted", + ], + "type": "object", }, }, }, "description": "Success", }, + "404": { + "description": "Task prompt not found", + }, }, - "summary": "List all skills", + "summary": "Soft-delete task prompt", "tags": [ - "Skills", + "Task Prompts", ], }, - "post": { + "get": { "parameters": [ { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "in": "query", - "name": "projectId", + "in": "path", + "name": "id", "required": true, "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", - "minLength": 1, "type": "string", }, }, ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateSkillInput", - }, - }, - }, - }, "responses": { - "201": { + "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SkillResponse", + "$ref": "#/components/schemas/TaskPromptResponse", }, }, }, "description": "Success", }, + "404": { + "description": "Task prompt not found", + }, }, - "summary": "Create or import a skill", + "summary": "Get task prompt", "tags": [ - "Skills", + "Task Prompts", ], }, }, - "/api/v1/skills/discover": { + "/api/v1/task-prompts/{id}/content": { "get": { "parameters": [ { - "in": "query", - "name": "source", - "required": true, - "schema": { - "type": "string", - }, - }, - { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "in": "query", - "name": "projectId", + "in": "path", + "name": "id", "required": true, "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", - "minLength": 1, "type": "string", }, }, @@ -14074,37 +16811,40 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/SkillDiscoveryResult", + "properties": { + "id": { + "type": "string", + }, + "text": { + "type": "string", + }, }, - "type": "array", + "required": [ + "id", + "text", + ], + "type": "object", }, }, }, "description": "Success", }, - "400": { - "description": "Missing or malformed source parameter", - }, "404": { - "description": "Repository not found", - }, - "502": { - "description": "GitHub API error", + "description": "Task prompt not found", }, }, - "summary": "Discover skills in a GitHub repository", + "summary": "Get resolved task prompt content", "tags": [ - "Skills", + "Task Prompts", ], }, }, - "/api/v1/skills/search": { - "get": { + "/api/v1/task-prompts/{id}/extract-features": { + "post": { "parameters": [ { - "in": "query", - "name": "q", + "in": "path", + "name": "id", "required": true, "schema": { "type": "string", @@ -14112,106 +16852,138 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, { "in": "query", - "name": "limit", + "name": "force", "required": false, "schema": { "type": "string", }, }, - { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "in": "query", - "name": "projectId", - "required": true, - "schema": { - "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", - "example": "00000000-0000-0000-0000-000000000000", - "minLength": 1, - "type": "string", + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "model": { + "type": "string", + }, + }, + "type": "object", + }, }, }, - ], + }, "responses": { "200": { "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/SkillSearchResult", + "properties": { + "cached": { + "type": "boolean", + }, + "features": { + "items": { + "$ref": "#/components/schemas/PromptFeatureResult", + }, + "type": "array", + }, + "featuresExtractedAt": { + "format": "date-time", + "type": [ + "string", + "null", + ], + }, + "suggestedFeatures": { + "items": { + "$ref": "#/components/schemas/SuggestedPromptFeature", + }, + "type": "array", + }, + "taskPromptId": { + "type": "string", + }, }, - "type": "array", + "required": [ + "taskPromptId", + "features", + "cached", + ], + "type": "object", }, }, }, "description": "Success", }, - "400": { - "description": "Missing query parameter", + "404": { + "description": "Task prompt not found", + }, + "503": { + "description": "LLM not configured", }, }, - "summary": "Search skills (internal + external)", + "summary": "Extract features from task prompt", "tags": [ - "Skills", + "Task Prompts", ], }, }, - "/api/v1/skills/search/external": { - "get": { + "/api/v1/task-prompts/{id}/features/{featureId}": { + "patch": { "parameters": [ { - "in": "query", - "name": "q", + "in": "path", + "name": "id", "required": true, "schema": { "type": "string", }, }, { - "in": "query", - "name": "limit", - "required": false, + "in": "path", + "name": "featureId", + "required": true, "schema": { "type": "string", }, }, - { - "description": "Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project.", - "in": "query", - "name": "projectId", - "required": false, - "schema": { - "description": "Optional project scope. When provided, external results are de-duplicated only against skills already installed in that project.", - "minLength": 1, - "type": "string", + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PatchTaskPromptFeatureInput", + }, }, }, - ], + }, "responses": { "200": { "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/SkillSearchResult", - }, - "type": "array", + "$ref": "#/components/schemas/TaskPromptResponse", }, }, }, "description": "Success", }, "400": { - "description": "Missing query parameter", + "description": "Invalid detected value", + }, + "404": { + "description": "Task prompt or feature not found", }, }, - "summary": "Search external skills registry", + "summary": "Toggle feature flag on task prompt", "tags": [ - "Skills", + "Task Prompts", ], }, }, - "/api/v1/skills/{id}(*)": { - "delete": { + "/api/v1/taxonomies": { + "get": { "parameters": [ { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", @@ -14227,19 +16999,26 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, ], "responses": { - "204": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/TaxonomyListItemResponse", + }, + "type": "array", + }, + }, + }, "description": "Success", }, - "404": { - "description": "Skill not found", - }, }, - "summary": "Soft-delete a skill", + "summary": "List project taxonomies", "tags": [ - "Skills", + "Taxonomies", ], }, - "get": { + "post": { "parameters": [ { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", @@ -14254,28 +17033,34 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateTaxonomyInput", + }, + }, + }, + }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SkillResponse", + "$ref": "#/components/schemas/TaxonomyResponse", }, }, }, "description": "Success", }, - "404": { - "description": "Skill not found", - }, }, - "summary": "Get skill by slug", + "summary": "Create a user taxonomy and revision 1", "tags": [ - "Skills", + "Taxonomies", ], }, }, - "/api/v1/skills/{id}(*)/resolve": { + "/api/v1/taxonomies/import": { "post": { "parameters": [ { @@ -14291,41 +17076,94 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` }, }, ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ImportTaxonomyYamlInput", + }, + }, + }, + }, "responses": { "201": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SkillRevisionResponse", + "$ref": "#/components/schemas/TaxonomyResponse", }, }, }, "description": "Success", }, - "404": { - "description": "Skill not found", + }, + "summary": "Import taxonomy YAML and referenced element YAML", + "tags": [ + "Taxonomies", + ], + }, + }, + "/api/v1/taxonomies/{id}": { + "delete": { + "responses": { + "204": { + "description": "Success", }, - "500": { - "description": "Blob storage error", + }, + "summary": "Soft-delete a taxonomy", + "tags": [ + "Taxonomies", + ], + }, + "get": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TaxonomyResponse", + }, + }, + }, + "description": "Success", }, }, - "summary": "Trigger skill resolution", + "summary": "Get a taxonomy", "tags": [ - "Skills", + "Taxonomies", + ], + }, + "patch": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateTaxonomyMetadataInput", + }, + }, + }, + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TaxonomyResponse", + }, + }, + }, + "description": "Success", + }, + }, + "summary": "Update taxonomy metadata", + "tags": [ + "Taxonomies", ], }, }, - "/api/v1/skills/{id}(*)/revisions": { + "/api/v1/taxonomies/{id}/detail": { "get": { "parameters": [ - { - "in": "query", - "name": "limit", - "required": false, - "schema": { - "type": "string", - }, - }, { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -14338,72 +17176,139 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "type": "string", }, }, + { + "in": "query", + "name": "revisionId", + "required": false, + "schema": { + "type": "string", + }, + }, ], "responses": { "200": { "content": { "application/json": { "schema": { - "items": { - "$ref": "#/components/schemas/SkillRevisionResponse", - }, - "type": "array", + "$ref": "#/components/schemas/TaxonomyDetailResponse", }, }, }, "description": "Success", }, - "404": { - "description": "Skill not found", + }, + "summary": "Get taxonomy detail with exact ordered element revisions", + "tags": [ + "Taxonomies", + ], + }, + }, + "/api/v1/taxonomies/{id}/restore": { + "post": { + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TaxonomyResponse", + }, + }, + }, + "description": "Success", }, }, - "summary": "List skill revisions", + "summary": "Restore a taxonomy", "tags": [ - "Skills", + "Taxonomies", ], }, }, - "/api/v1/task-prompts": { + "/api/v1/taxonomies/{id}/revisions": { "get": { - "parameters": [ - { - "in": "query", - "name": "limit", - "required": false, - "schema": { - "type": [ - "number", - "null", - ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/TaxonomyRevisionResponse", + }, + "type": "array", + }, + }, }, + "description": "Success", }, - { - "in": "query", - "name": "offset", - "required": false, - "schema": { - "type": [ - "number", - "null", - ], + }, + "summary": "List taxonomy revisions", + "tags": [ + "Taxonomies", + ], + }, + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateTaxonomyRevisionInput", + }, }, }, - { - "in": "query", - "name": "search", - "required": false, - "schema": { - "type": "string", + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TaxonomyRevisionResponse", + }, + }, }, + "description": "Success", }, - { - "in": "query", - "name": "type", - "required": false, - "schema": { - "$ref": "#/components/schemas/PromptType", + }, + "summary": "Create taxonomy revision", + "tags": [ + "Taxonomies", + ], + }, + }, + "/api/v1/taxonomy-element-revisions/{revisionId}": { + "get": { + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TaxonomyElementRevisionResponse", + }, + }, }, + "description": "Success", + }, + }, + "summary": "Get an exact taxonomy element revision", + "tags": [ + "Taxonomies", + ], + }, + }, + "/api/v1/taxonomy-element-revisions/{revisionId}/export": { + "get": { + "responses": { + "200": { + "description": "Success", }, + }, + "summary": "Export an exact taxonomy element revision as portable YAML", + "tags": [ + "Taxonomies", + ], + }, + }, + "/api/v1/taxonomy-elements": { + "get": { + "parameters": [ { "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", "in": "query", @@ -14422,39 +17327,19 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "properties": { - "items": { - "items": { - "$ref": "#/components/schemas/TaskPromptResponse", - }, - "type": "array", - }, - "limit": { - "type": "number", - }, - "offset": { - "type": "number", - }, - "total": { - "type": "number", - }, - }, - "required": [ - "items", - "total", - "limit", - "offset", - ], - "type": "object", + "items": { + "$ref": "#/components/schemas/TaxonomyElementResponse", + }, + "type": "array", }, }, }, "description": "Success", }, }, - "summary": "List task prompts", + "summary": "List project taxonomy elements", "tags": [ - "Task Prompts", + "Taxonomies", ], }, "post": { @@ -14476,7 +17361,7 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CreateTaskPromptInput", + "$ref": "#/components/schemas/CreateTaxonomyElementInput", }, }, }, @@ -14486,296 +17371,179 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TaskPromptResponse", + "$ref": "#/components/schemas/TaxonomyElementResponse", }, }, }, "description": "Success", }, - "400": { - "description": "Empty text string", - }, }, - "summary": "Create or find task prompt", + "summary": "Create a user taxonomy element and revision 1", "tags": [ - "Task Prompts", + "Taxonomies", ], }, }, - "/api/v1/task-prompts/generate": { - "post": { - "requestBody": { - "content": { - "application/json": { - "schema": { - "properties": { - "description": { - "type": "string", - }, - "existingPrompt": { - "type": "string", - }, - }, - "type": "object", - }, - }, + "/api/v1/taxonomy-elements/{id}": { + "delete": { + "responses": { + "204": { + "description": "Success", }, }, + "summary": "Soft-delete a taxonomy element", + "tags": [ + "Taxonomies", + ], + }, + "get": { "responses": { "200": { "content": { "application/json": { "schema": { - "properties": { - "tasks": { - "items": { - "type": "string", - }, - "type": "array", - }, - }, - "required": [ - "tasks", - ], - "type": "object", + "$ref": "#/components/schemas/TaxonomyElementResponse", }, }, }, "description": "Success", }, - "503": { - "description": "LLM not configured", - }, }, - "summary": "Generate task prompts", + "summary": "Get a taxonomy element", "tags": [ - "Task Prompts", + "Taxonomies", ], }, - }, - "/api/v1/task-prompts/{id}": { - "delete": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", + "patch": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateTaxonomyMetadataInput", + }, }, }, - ], + }, "responses": { "200": { "content": { "application/json": { "schema": { - "properties": { - "deleted": { - "type": "boolean", - }, - }, - "required": [ - "deleted", - ], - "type": "object", + "$ref": "#/components/schemas/TaxonomyElementResponse", }, }, }, "description": "Success", }, - "404": { - "description": "Task prompt not found", - }, }, - "summary": "Soft-delete task prompt", + "summary": "Update taxonomy element metadata", "tags": [ - "Task Prompts", + "Taxonomies", ], }, - "get": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - ], + }, + "/api/v1/taxonomy-elements/{id}/restore": { + "post": { "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TaskPromptResponse", + "$ref": "#/components/schemas/TaxonomyElementResponse", }, }, }, "description": "Success", }, - "404": { - "description": "Task prompt not found", - }, }, - "summary": "Get task prompt", + "summary": "Restore a taxonomy element", "tags": [ - "Task Prompts", + "Taxonomies", ], }, }, - "/api/v1/task-prompts/{id}/content": { + "/api/v1/taxonomy-elements/{id}/revisions": { "get": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - ], "responses": { "200": { "content": { "application/json": { "schema": { - "properties": { - "id": { - "type": "string", - }, - "text": { - "type": "string", - }, + "items": { + "$ref": "#/components/schemas/TaxonomyElementRevisionResponse", }, - "required": [ - "id", - "text", - ], - "type": "object", + "type": "array", }, }, }, "description": "Success", }, - "404": { - "description": "Task prompt not found", - }, }, - "summary": "Get resolved task prompt content", + "summary": "List taxonomy element revisions", "tags": [ - "Task Prompts", + "Taxonomies", ], }, - }, - "/api/v1/task-prompts/{id}/extract-features": { "post": { - "parameters": [ - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - { - "in": "query", - "name": "force", - "required": false, - "schema": { - "type": "string", - }, - }, - ], "requestBody": { "content": { "application/json": { "schema": { - "properties": { - "model": { - "type": "string", - }, + "$ref": "#/components/schemas/CreateTaxonomyElementRevisionInput", + }, + }, + }, + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TaxonomyElementRevisionResponse", }, - "type": "object", }, }, + "description": "Success", }, }, + "summary": "Create taxonomy element revision", + "tags": [ + "Taxonomies", + ], + }, + }, + "/api/v1/taxonomy-revisions/{revisionId}": { + "get": { "responses": { "200": { "content": { "application/json": { "schema": { - "properties": { - "cached": { - "type": "boolean", - }, - "features": { - "items": { - "$ref": "#/components/schemas/PromptFeatureResult", - }, - "type": "array", - }, - "featuresExtractedAt": { - "format": "date-time", - "type": [ - "string", - "null", - ], - }, - "suggestedFeatures": { - "items": { - "$ref": "#/components/schemas/SuggestedPromptFeature", - }, - "type": "array", - }, - "taskPromptId": { - "type": "string", - }, - }, - "required": [ - "taskPromptId", - "features", - "cached", - ], - "type": "object", + "$ref": "#/components/schemas/TaxonomyRevisionResponse", }, }, }, "description": "Success", }, - "404": { - "description": "Task prompt not found", - }, - "503": { - "description": "LLM not configured", - }, }, - "summary": "Extract features from task prompt", + "summary": "Get an exact taxonomy revision", "tags": [ - "Task Prompts", + "Taxonomies", ], }, }, - "/api/v1/task-prompts/{id}/features/{featureId}": { - "patch": { + "/api/v1/taxonomy-revisions/{revisionId}/copy": { + "post": { "parameters": [ { - "in": "path", - "name": "id", - "required": true, - "schema": { - "type": "string", - }, - }, - { - "in": "path", - "name": "featureId", + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "in": "query", + "name": "projectId", "required": true, "schema": { + "description": "Project scope. Required on scoped list and root-create operations; requests without a resolvable project are rejected with 400. There is no default project.", + "example": "00000000-0000-0000-0000-000000000000", + "minLength": 1, "type": "string", }, }, @@ -14784,32 +17552,39 @@ exports[`OpenAPI spec snapshot > matches the committed snapshot 1`] = ` "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PatchTaskPromptFeatureInput", + "$ref": "#/components/schemas/CopyBuiltinTaxonomyRevisionInput", }, }, }, }, "responses": { - "200": { + "201": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TaskPromptResponse", + "$ref": "#/components/schemas/TaxonomyResponse", }, }, }, "description": "Success", }, - "400": { - "description": "Invalid detected value", - }, - "404": { - "description": "Task prompt or feature not found", + }, + "summary": "Copy an exact built-in revision to user revision 1", + "tags": [ + "Taxonomies", + ], + }, + }, + "/api/v1/taxonomy-revisions/{revisionId}/export": { + "get": { + "responses": { + "200": { + "description": "Success", }, }, - "summary": "Toggle feature flag on task prompt", + "summary": "Export an exact taxonomy revision as portable YAML", "tags": [ - "Task Prompts", + "Taxonomies", ], }, }, diff --git a/apps/api/src/experiment-coordinator.test.ts b/apps/api/src/experiment-coordinator.test.ts index 2fab1d4e2..753f812b5 100644 --- a/apps/api/src/experiment-coordinator.test.ts +++ b/apps/api/src/experiment-coordinator.test.ts @@ -105,6 +105,64 @@ afterEach(() => { }); describe("ExperimentCoordinator", () => { + it("projects the registered Taxonome handler as the readout dependency", () => { + const coordinator = new ExperimentCoordinator( + collections({ requests: [], services: [] }), + ); + const request = { + _id: "request-1", + projectId: "project-1", + scenario: { task: "task", criteria: [] }, + workerType: "worker", + createdAt: now, + priority: 0, + run: { + _id: "run-1", + attemptNumber: 1, + status: "done", + handlerStatus: { + "pp-taxonomy": { + status: "done", + version: 1, + updatedAt: now, + }, + }, + }, + } as RequestDocument; + const handler = { + _id: "pp-taxonomy", + type: "post-process-handler", + version: 1, + queue: "post-processor-queue", + selector: "taxonomy", + autoBackfill: true, + dependsOn: ["pp-atif"], + } as ServiceHandlerDocument; + + const projected = ( + coordinator as unknown as { + projectRequest( + requestId: string, + runId: string, + request: RequestDocument, + handlers: ServiceHandlerDocument[], + ): ExperimentRequestState; + } + ).projectRequest(request._id, request.run!._id, request, [handler]); + + expect(projected).toEqual({ + requestId: "request-1", + runId: "run-1", + dependencies: [{ + handlerId: "pp-taxonomy", + state: "ready", + version: 1, + updatedAt: now, + }], + blockers: [], + }); + }); + it("queues ready Experiment reports independently from unused linked templates", async () => { const requestState: ExperimentRequestState = { requestId: "request-1", diff --git a/apps/api/src/experiment-coordinator.ts b/apps/api/src/experiment-coordinator.ts index 0c98447cb..4a7bbc440 100644 --- a/apps/api/src/experiment-coordinator.ts +++ b/apps/api/src/experiment-coordinator.ts @@ -487,10 +487,14 @@ export class ExperimentCoordinator { } const experimentHandlers = request.run.handlerStatus ?? {}; const dependencyIds = handlers - .filter((handler) => handler.selector === "observations") + .filter( + (handler) => + handler.selector === "taxonomy" || + handler.selector === "observations", + ) .map((handler) => handler._id); const effectiveDependencyIds = - dependencyIds.length > 0 ? dependencyIds : ["pp-observations"]; + dependencyIds.length > 0 ? dependencyIds : ["pp-taxonomy"]; const dependencies = effectiveDependencyIds.map((handlerId) => { const source = experimentHandlers[handlerId]; const state = namedHandlerState( diff --git a/apps/api/src/experiment-lifecycle.test.ts b/apps/api/src/experiment-lifecycle.test.ts index d4eb067b3..bb7d0f705 100644 --- a/apps/api/src/experiment-lifecycle.test.ts +++ b/apps/api/src/experiment-lifecycle.test.ts @@ -14,6 +14,7 @@ import { type ExperimentRequestState, type ReportDocument, type RequestDocument, + taxonomyHandlerSummary, } from "shared"; import { afterEach, describe, expect, it, vi } from "vitest"; import { @@ -196,6 +197,8 @@ describe("ExperimentLifecycleCoordinator", () => { it.each([ { status: "done", handlerStatus: { "pp-atif": { status: "queued" } } }, + { status: "done", handlerStatus: { "pp-taxonomy": { status: "queued" } } }, + { status: "done", handlerStatus: { "pp-taxonomy": { status: "processing" } } }, { status: "done", reportsTriggerCompletedAt: undefined }, { status: "done", reportPublicationIds: ["publishing-report"] }, ])("rejects unfinished run work $handlerStatus $reportPublicationIds", async (run) => { @@ -206,6 +209,47 @@ describe("ExperimentLifecycleCoordinator", () => { expect(db.experiments.findOneAndUpdate).not.toHaveBeenCalled(); }); + it("keeps derived taxonomy pending view-only while deletion remains fenced by durable work", async () => { + const current = root(); + const run: NonNullable = { + _id: "run-1", + status: "done", + attemptNumber: 1, + reportsTriggerCompletedAt: undefined, + }; + const request = { + _id: "request-1", + run, + }; + const db = collections({ current, updated: current, requests: [request] }); + vi.spyOn(FixedPayloadStore.prototype, "readRequests").mockResolvedValue([requestState]); + + expect(taxonomyHandlerSummary(request.run, true)).toEqual({ + status: "pending", + resultAvailable: false, + }); + expect(request.run.handlerStatus).toBeUndefined(); + await expect( + new ExperimentLifecycleCoordinator(db).startDelete( + "project-1", + "experiment-1", + ), + ).rejects.toMatchObject({ + status: 409, + conflicts: expect.arrayContaining([ + expect.objectContaining({ + requestId: "request-1", + reason: "report-trigger-incomplete", + }), + ]), + }); + expect(request.run).toMatchObject({ + status: "done", + attemptNumber: 1, + }); + expect(request.run.handlerStatus).toBeUndefined(); + }); + it.each(["pending", "generating"])("rejects an outstanding %s report by exact report ID", async (status) => { const current = root(); const db = collections({ current, updated: current, requests: [{ _id: "request-1" }], reports: [{ _id: "report-busy", status }] }); @@ -317,9 +361,35 @@ describe("ExperimentLifecycleCoordinator", () => { const deleted = { ...root(lifecycle("deleted", targets)), deletedAt: now }; const restoring = { ...root(lifecycle("restoring", targets)), deletedAt: now }; const active = root(); + const taxonomy = { + taxonomyRevisionId: "taxonomy-revision-1", + ref: "ax-taxonomy@r1", + contentSha256: "a".repeat(64), + }; + for (const experiment of [deleting, deleted, restoring, active]) { + experiment.revision.taxonomy = taxonomy; + experiment.execution.taxonomy = taxonomy; + } const db = collections({ current: deleting, updated: deleted, - requests: [{ _id: "request-1", run: { _id: "run-1", outcome, result: "retained result", turns: [{ iteration: 1 }] } }], + requests: [{ + _id: "request-1", + run: { + _id: "run-1", + outcome, + result: "retained result", + turns: [{ iteration: 1 }], + handlerStatus: { + "pp-taxonomy": { status: "done", updatedAt: now }, + }, + taxonomyEvaluation: { + url: "https://storage.invalid/taxonomy.json.gz", + schemaVersion: "scope/taxonomy-run-result/v1", + contentSha256: "b".repeat(64), + sizeBytes: 123, + }, + }, + }], }); vi.spyOn(ExperimentLifecycleTargetStore.prototype, "read").mockResolvedValue(targets); vi.spyOn(FixedPayloadStore.prototype, "readRequests").mockResolvedValue([requestState]); @@ -338,6 +408,7 @@ describe("ExperimentLifecycleCoordinator", () => { expect(after.deletionProvenance).toBeUndefined(); expect(restored.lifecycle).toBeUndefined(); expect(restored.execution.experimentExecutionId).toBe("execution-1"); + expect(restored.execution.taxonomy).toEqual(taxonomy); expect(rollover).not.toHaveBeenCalled(); // Successful completion removes the intent, so a new deletion can claim it. diff --git a/apps/api/src/experiment-membership.ts b/apps/api/src/experiment-membership.ts index 99e486dc8..9c4f2f046 100644 --- a/apps/api/src/experiment-membership.ts +++ b/apps/api/src/experiment-membership.ts @@ -77,6 +77,7 @@ export class ExperimentMembershipCoordinator { metadata: ExperimentMetadata; requestIds: readonly string[]; reportTemplateIds?: readonly string[]; + taxonomy?: import("shared").ExperimentTaxonomySelection; now?: Date; }): Promise { const requestIds = uniqueCanonicalRequestIds(input.requestIds); @@ -163,7 +164,9 @@ export class ExperimentMembershipCoordinator { metadata: input.metadata, requestIds, reportTemplateIds, + taxonomy: input.taxonomy, }), + ...(input.taxonomy ? { taxonomy: input.taxonomy } : {}), createdAt: now, }, experimentExecutionId: input.experimentExecutionId, diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 02bdff2a0..bd621aa2b 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -9,10 +9,10 @@ import { DefaultAzureCredential } from "@azure/identity"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import dotenv from "dotenv"; -import { TaskPromptStore, SkillRevisionStore, SkillResolver, CodebaseStore, CodebaseRevisionStore, CodebaseResolver, ResourceStore, ResourceRevisionStore, ResourceResolver, McpSecretClient, McpSecretUnavailableError, BlobStorage, RedisHeartbeatStore, ProjectStore, loadAuthConfigFromEnv } from "shared"; +import { TaskPromptStore, SkillRevisionStore, SkillResolver, CodebaseStore, CodebaseRevisionStore, CodebaseResolver, ResourceStore, ResourceRevisionStore, ResourceResolver, TaxonomyRootStore, TaxonomyRevisionStore, McpSecretClient, McpSecretUnavailableError, BlobStorage, RedisHeartbeatStore, ProjectStore, loadAuthConfigFromEnv } from "shared"; import type { ExperimentDocument, ExperimentExecutionHistoryDocument } from "shared"; import { initTelemetry } from "telemetry"; -import type { TaskPromptDocument, SkillDocument, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, ResourceDocument, ResourceRevisionDocument, ProfileDocument, ProfileVersionDocument, ProjectDocument, HeartbeatStore, AuthProvider, ProfileEnricher, UserDocument } from "shared"; +import type { TaskPromptDocument, SkillDocument, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, ResourceDocument, ResourceRevisionDocument, TaxonomyDocument, TaxonomyRevisionDocument, TaxonomyElementDocument, TaxonomyElementRevisionDocument, PortableTaxonomyDocument, PortableTaxonomyElementDocument, ProfileDocument, ProfileVersionDocument, ProjectDocument, HeartbeatStore, AuthProvider, ProfileEnricher, UserDocument } from "shared"; import { UserStore } from "./auth/user-store.js"; import { createAuthMiddleware, createUserAccessMiddleware } from "./auth/middleware.js"; import { authErrorHandler } from "./auth/error-handler.js"; @@ -32,6 +32,7 @@ import { registerRequestsVideoRoutes } from "./routes/requests/video.js"; import { registerRequestsToolCallsRoutes } from "./routes/requests/tool-calls.js"; import { registerRequestsSnapshotsRoutes } from "./routes/requests/snapshots.js"; import { registerRequestsArchiveRoutes } from "./routes/requests/archive.js"; +import { registerRequestsTaxonomyEvaluationRoutes } from "./routes/requests/taxonomy-evaluation.js"; import { registerCriteriaRoutes } from "./routes/criteria.js"; import { registerPromptFeaturesRoutes } from "./routes/prompt-features.js"; import { registerTaskPromptsRoutes } from "./routes/task-prompts.js"; @@ -49,6 +50,8 @@ import { registerSecretsRoutes } from "./routes/secrets.js"; import { registerProfilesRoutes } from "./routes/profiles.js"; import { registerProjectsRoutes } from "./routes/projects.js"; import { registerExperimentsRoutes } from "./routes/experiments.js"; +import { registerTaxonomyRoutes } from "./routes/taxonomies.js"; +import { registerTaxonomyReadoutRoutes } from "./routes/taxonomy-readout.js"; import { ProjectScopeError } from "./utils/project-scope.js"; import { registerUsersRoutes } from "./routes/users.js"; import { shutdownPortalCopilotClientPool } from "./copilot-client-pool.js"; @@ -144,6 +147,14 @@ let resourceRevisionCollection: Collection; let resourceStore: ResourceStore; let resourceRevisionStore: ResourceRevisionStore; let resourceResolver: ResourceResolver; +let taxonomyCollection: Collection; +let taxonomyRevisionCollection: Collection; +let taxonomyElementCollection: Collection; +let taxonomyElementRevisionCollection: Collection; +let taxonomyStore: TaxonomyRootStore; +let taxonomyRevisionStore: TaxonomyRevisionStore; +let taxonomyElementStore: TaxonomyRootStore; +let taxonomyElementRevisionStore: TaxonomyRevisionStore; let blobStorage: BlobStorage; let heartbeatStore: HeartbeatStore; let reportQueueClient: QueueClient; @@ -237,6 +248,23 @@ async function initializeClients(): Promise { resourceRevisionStore = new ResourceRevisionStore(resourceRevisionCollection, resourceStore); resourceResolver = new ResourceResolver(); + taxonomyCollection = db.collection("taxonomies"); + taxonomyRevisionCollection = db.collection("taxonomy-revisions"); + taxonomyElementCollection = db.collection("taxonomy-elements"); + taxonomyElementRevisionCollection = db.collection("taxonomy-element-revisions"); + taxonomyStore = new TaxonomyRootStore(taxonomyCollection); + taxonomyRevisionStore = new TaxonomyRevisionStore( + taxonomyRevisionCollection, + taxonomyStore, + "taxonomyId", + ); + taxonomyElementStore = new TaxonomyRootStore(taxonomyElementCollection); + taxonomyElementRevisionStore = new TaxonomyRevisionStore( + taxonomyElementRevisionCollection, + taxonomyElementStore, + "elementId", + ); + // Note: Collection indexes are managed by db-migrations (see 002-create-indexes.ts). // Run `pnpm migrate:up` to apply pending migrations. @@ -253,6 +281,7 @@ async function initializeClients(): Promise { // when ready. See apps/portal/src/lib/gates.ts (GATE_FEATURE_FLAGS). { key: "gates-run", label: "Run Gate", enabled: false }, { key: "gates-deploy", label: "Deploy Gate", enabled: false }, + { key: "taxonome", label: "Taxonome and Taxonomies", enabled: false }, ]; for (const flag of defaultFlags) { await featureFlagCollection.updateOne( @@ -344,6 +373,14 @@ const routeCtx: RouteContext = { get resourceStore() { return resourceStore; }, get resourceRevisionStore() { return resourceRevisionStore; }, get resourceResolver() { return resourceResolver; }, + get taxonomyCollection() { return taxonomyCollection; }, + get taxonomyRevisionCollection() { return taxonomyRevisionCollection; }, + get taxonomyElementCollection() { return taxonomyElementCollection; }, + get taxonomyElementRevisionCollection() { return taxonomyElementRevisionCollection; }, + get taxonomyStore() { return taxonomyStore; }, + get taxonomyRevisionStore() { return taxonomyRevisionStore; }, + get taxonomyElementStore() { return taxonomyElementStore; }, + get taxonomyElementRevisionStore() { return taxonomyElementRevisionStore; }, get projectStore() { return projectStore; }, get reportQueueClient() { return reportQueueClient; }, get blobStorage() { return blobStorage; }, @@ -371,6 +408,8 @@ registerSecretsRoutes(routeCtx); registerSystemRoutes(routeCtx); registerProjectsRoutes(routeCtx); registerExperimentsRoutes(routeCtx); +registerTaxonomyRoutes(routeCtx); +registerTaxonomyReadoutRoutes(routeCtx); registerRequestsRoutes(routeCtx); registerRequestsCancelRoutes(routeCtx); registerRequestsLogsRoutes(routeCtx); @@ -380,6 +419,7 @@ registerRequestsVideoRoutes(routeCtx); registerRequestsToolCallsRoutes(routeCtx); registerRequestsSnapshotsRoutes(routeCtx); registerRequestsArchiveRoutes(routeCtx); +registerRequestsTaxonomyEvaluationRoutes(routeCtx); registerCriteriaRoutes(routeCtx); registerPromptFeaturesRoutes(routeCtx); registerTaskPromptsRoutes(routeCtx); diff --git a/apps/api/src/route-context.ts b/apps/api/src/route-context.ts index 43e3db777..62722a11f 100644 --- a/apps/api/src/route-context.ts +++ b/apps/api/src/route-context.ts @@ -55,6 +55,12 @@ import type { RunHistoryDocumentSchema, ExperimentDocument, ExperimentExecutionHistoryDocument, + TaxonomyDocument, + TaxonomyElementDocument, + TaxonomyRevisionDocument, + TaxonomyElementRevisionDocument, + TaxonomyRootStore, + TaxonomyRevisionStore, } from "shared"; // ─── Document types (inferred from Zod schemas) ───────────────────────────── @@ -118,6 +124,10 @@ export interface RouteContext { codebaseRevisionCollection: Collection; resourceCollection: Collection; resourceRevisionCollection: Collection; + taxonomyCollection: Collection; + taxonomyRevisionCollection: Collection; + taxonomyElementCollection: Collection; + taxonomyElementRevisionCollection: Collection; // Services taskPromptStore: TaskPromptStore; @@ -129,6 +139,18 @@ export interface RouteContext { resourceStore: ResourceStore; resourceRevisionStore: ResourceRevisionStore; resourceResolver: ResourceResolver; + taxonomyStore: TaxonomyRootStore; + taxonomyRevisionStore: TaxonomyRevisionStore< + TaxonomyRevisionDocument, + TaxonomyDocument, + import("shared").PortableTaxonomyDocument + >; + taxonomyElementStore: TaxonomyRootStore; + taxonomyElementRevisionStore: TaxonomyRevisionStore< + TaxonomyElementRevisionDocument, + TaxonomyElementDocument, + import("shared").PortableTaxonomyElementDocument + >; projectStore: ProjectStore; // Token Manager client (null when TOKEN_MANAGER_URL not set) diff --git a/apps/api/src/routes/experiments.test.ts b/apps/api/src/routes/experiments.test.ts index 79204c266..862ede63f 100644 --- a/apps/api/src/routes/experiments.test.ts +++ b/apps/api/src/routes/experiments.test.ts @@ -10,7 +10,7 @@ import { } from "@asteasolutions/zod-to-openapi"; import { describe, expect, it, vi } from "vitest"; import { z } from "zod"; -import type { RouteContext } from "../route-context.js"; +import type { RequestDocument, RouteContext } from "../route-context.js"; import { ExperimentStore, type ExperimentDocument } from "shared"; import { ExperimentLifecycleCoordinator, @@ -339,20 +339,48 @@ describe("Experiment API contract", () => { it("inserts composed requests before publishing their exact revision", async () => { const experimentCollection = emptyExperimentCollection(); - const insertMany = vi.fn().mockResolvedValue({ - acknowledged: true, - insertedCount: 1, + let insertedDocuments: RequestDocument[] = []; + const insertMany = vi.fn(async (documents: RequestDocument[]) => { + insertedDocuments = structuredClone(documents); + return { + acknowledged: true, + insertedCount: documents.length, + }; }); const updateMany = vi.fn().mockResolvedValue({ acknowledged: true, matchedCount: 1, modifiedCount: 1, }); + const find = vi.fn(() => ({ + toArray: vi.fn(async () => structuredClone(insertedDocuments)), + })); const { app } = harness({ experimentCollection, experimentExecutionsCollection: {}, experimentChunksCollection: {}, - requestCollection: { insertMany, updateMany }, + requestCollection: { insertMany, updateMany, find }, + runsCollection: { find: vi.fn() }, + agentCollection: { + findOne: vi.fn().mockResolvedValue({ + _id: "coder", + name: "Coder", + available: true, + supportedModels: [], + versions: [{ + agentVersion: "coder-v1", + workerVersion: "coder-v1-build", + components: {}, + gitCommit: "abcdef0", + buildTime: "20261029T000000Z", + imageTag: "coder-v1-build", + queueName: "queue-coder", + status: "active", + createdAt: new Date(), + }], + createdAt: new Date(), + }), + }, taskPromptStore: { findOrCreate: vi.fn().mockResolvedValue({ _id: "prompt-1" }), }, @@ -385,6 +413,7 @@ describe("Experiment API contract", () => { experimentExpectedRequestCount: 1, experimentRequestIndex: 0, experimentInitialRunId: expect.any(String), + agentVersion: "coder-v1", run: { _id: expect.any(String), status: "pending", @@ -399,6 +428,180 @@ describe("Experiment API contract", () => { ); }); + it("resolves and persists every mixed-model profile target before publication", async () => { + const experimentCollection = emptyExperimentCollection(); + let insertedDocuments: RequestDocument[] = []; + const insertMany = vi.fn(async (documents: RequestDocument[]) => { + insertedDocuments = structuredClone(documents); + return { + acknowledged: true, + insertedCount: documents.length, + }; + }); + const find = vi.fn(() => ({ + toArray: vi.fn(async () => structuredClone(insertedDocuments)), + })); + const findAgent = vi.fn().mockResolvedValue({ + _id: "coder-acp-copilot", + name: "GitHub Copilot CLI", + available: true, + supportedModels: ["claude-haiku-4.5", "gpt-5.6-sol-fast"], + versions: [{ + agentVersion: "copilot-dev", + workerVersion: "copilot-dev-build", + components: {}, + gitCommit: "abcdef0", + buildTime: "20261029T000000Z", + imageTag: "copilot-dev-build", + queueName: "queue-coder-acp-copilot", + status: "active", + createdAt: new Date(), + }], + createdAt: new Date(), + }); + const requests = Array.from({ length: 6 }, (_, index) => ({ + scenario: { task: "Build it", criteria: ["works"] }, + workerType: "coder-acp-copilot", + model: "client-value-must-not-control-profile-composition", + profileId: index < 3 ? "profile-a@1" : "profile-b@2", + })); + const { app } = harness({ + experimentCollection, + experimentExecutionsCollection: {}, + experimentChunksCollection: {}, + requestCollection: { + insertMany, + updateMany: vi.fn().mockResolvedValue({ + acknowledged: true, + matchedCount: requests.length, + modifiedCount: requests.length, + }), + find, + }, + runsCollection: { find: vi.fn() }, + profileCollection: { + findOne: vi.fn(async (filter: { _id: string }) => ({ + _id: filter._id, + projectId: "project-1", + name: filter._id, + latestVersion: filter._id === "profile-a" ? 1 : 2, + createdAt: new Date(), + })), + }, + profileVersionCollection: { + findOne: vi.fn(async (filter: { profileId: string; version: number }) => + filter.profileId === "profile-a" + ? { + _id: "profile-version-a", + ref: "profile-a@1", + profileId: "profile-a", + projectId: "project-1", + version: 1, + workerType: "coder-acp-copilot", + model: "claude-haiku-4.5", + agentVersion: "copilot-dev", + mcpServers: ["ms-learn"], + createdAt: new Date(), + } + : { + _id: "profile-version-b", + ref: "profile-b@2", + profileId: "profile-b", + projectId: "project-1", + version: 2, + workerType: "coder-acp-copilot", + model: "gpt-5.6-sol-fast", + agentVersion: "copilot-dev", + skillRevisions: ["owner/skill@commit"], + createdAt: new Date(), + } + ), + }, + agentCollection: { findOne: findAgent }, + taskPromptStore: { + findOrCreate: vi.fn().mockResolvedValue({ _id: "prompt-1" }), + }, + blobStorage: { + getLogsBlobUrl: vi.fn( + (path: string) => `https://storage.invalid/${path}`, + ), + }, + } as unknown as Partial); + + const response = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "compose", + metadata: { name: "Mixed targets" }, + requests, + }); + + expect(response.status).toBe(201); + expect(response.body.requestIds).toHaveLength(6); + expect(insertedDocuments).toHaveLength(6); + expect(insertedDocuments.map((document) => document.agentVersion)).toEqual( + Array(6).fill("copilot-dev"), + ); + expect(insertedDocuments.map((document) => document.model)).toEqual([ + "claude-haiku-4.5", + "claude-haiku-4.5", + "claude-haiku-4.5", + "gpt-5.6-sol-fast", + "gpt-5.6-sol-fast", + "gpt-5.6-sol-fast", + ]); + expect(insertedDocuments.map((document) => document.profileId)).toEqual([ + "profile-a@1", + "profile-a@1", + "profile-a@1", + "profile-b@2", + "profile-b@2", + "profile-b@2", + ]); + expect(insertedDocuments.map((document) => document.mcpServers)).toEqual([ + ["ms-learn"], + ["ms-learn"], + ["ms-learn"], + undefined, + undefined, + undefined, + ]); + expect(insertedDocuments.map((document) => document.skillRevisions)).toEqual([ + undefined, + undefined, + undefined, + ["owner/skill@commit"], + ["owner/skill@commit"], + ["owner/skill@commit"], + ]); + expect(findAgent).toHaveBeenCalledTimes(2); + }); + + it("rejects criterion-free multi-turn composition before inserting requests", async () => { + const insertMany = vi.fn(); + const { app } = harness({ + experimentCollection: emptyExperimentCollection(), + requestCollection: { insertMany }, + } as unknown as Partial); + + const response = await request(app) + .post("/api/v1/experiments?projectId=project-1") + .send({ + mode: "compose", + metadata: { name: "Invalid multi-turn" }, + requests: [{ + scenario: { task: "Build it", criteria: [] }, + workerType: "coder-acp-copilot", + }], + }); + + expect(response.status).toBe(400); + expect(response.body.error).toContain( + "At least one criterion is required", + ); + expect(insertMany).not.toHaveBeenCalled(); + }); + it("publishes only the canonical routes and public execution fields", () => { const { registry } = harness(); const document = generateDocument(registry); @@ -432,6 +635,8 @@ describe("Experiment API contract", () => { type: "object", properties: { experimentExecutionId: { type: "string" }, + requestCount: { type: "integer", minimum: 0 }, + requests: { type: "array" }, readiness: expect.any(Object), blockers: expect.any(Object), progress: expect.any(Object), @@ -445,6 +650,335 @@ describe("Experiment API contract", () => { ); }); + it("hydrates current execution members in exact vector order with bounded reads", async () => { + const now = new Date("2026-10-29T21:57:18.488Z"); + const states = [ + { requestId: "request-active", runId: "run-active", dependencies: [], blockers: [] }, + { requestId: "request-deleted", runId: "run-deleted", dependencies: [], blockers: [] }, + { requestId: "request-missing", runId: "run-missing", dependencies: [], blockers: [] }, + ]; + const execution = { + experimentExecutionId: "execution-1", + attemptNumber: 1, + version: 3, + requests: { + storage: "inline" as const, + entryCount: states.length, + payloadFingerprint: "payload", + payloadVersion: 1, + entries: states, + }, + vectorFingerprint: "vector", + taxonomy: { + taxonomyRevisionId: "taxonomy-revision-1", + ref: "ax-taxonomy@r1", + contentSha256: "b".repeat(64), + }, + handlerSummary: {}, + blocking: [], + createdAt: now, + updatedAt: now, + }; + const experiment = { + _id: "experiment-1", + experimentId: "experiment-1", + projectId: "project-1", + storageProfileVersion: "cosmos-mongo-2mb-v1", + stateVersion: 1, + metadata: { name: "Hydration test" }, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: states.length, + membershipFingerprint: "membership", + definitionFingerprint: "definition", + reportTemplateIds: [], + createdAt: now, + }, + execution, + createdAt: now, + updatedAt: now, + finalizedAt: now, + } as ExperimentDocument; + const get = vi + .spyOn(ExperimentStore.prototype, "get") + .mockResolvedValue(experiment); + const readExecutionRequests = vi + .spyOn(ExperimentStore.prototype, "readExecutionRequests") + .mockResolvedValue(states); + const requestToArray = vi.fn().mockResolvedValue([ + { + _id: "request-active", + projectId: "project-1", + scenario: { task: "Active task", criteria: [] }, + taskPromptId: "prompt-active", + workerType: "coder-acp-copilot", + model: "model-a", + reasoningEffort: "high", + agentVersion: "copilot-dev", + mcpServers: ["github"], + skillRevisions: ["owner/skill@r1"], + extensions: ["ms-python.python@1.0.0"], + profileId: "profile-a@1", + priority: 5, + run: { + _id: "run-active", + attemptNumber: 1, + status: "processing", + os: { platform: "linux", release: "6.8", arch: "arm64" }, + worker: { instanceId: "worker-1", podName: "pod-1" }, + lastHeartbeatAt: now, + startedAt: now, + handlerStatus: { + "pp-taxonomy": { + status: "done", + version: "1", + updatedAt: now, + claimToken: "claim-1", + }, + }, + taxonomyEvaluation: { + url: "https://storage.example/result.json.gz", + schemaVersion: "2", + contentSha256: "a".repeat(64), + sizeBytes: 123, + }, + }, + }, + { + _id: "request-deleted", + projectId: "project-1", + scenario: { task: "Deleted task", criteria: [] }, + workerType: "coder-acp-copilot", + model: "model-b", + deletedAt: now, + run: { + _id: "run-deleted", + attemptNumber: 1, + status: "done", + outcome: "succeeded", + }, + }, + ]); + const requestFind = vi.fn(() => ({ toArray: requestToArray })); + const runsToArray = vi.fn().mockResolvedValue([]); + const runsFind = vi.fn(() => ({ toArray: runsToArray })); + const { app } = harness({ + requestCollection: { find: requestFind }, + runsCollection: { find: runsFind }, + } as unknown as Partial); + + const response = await request(app).get( + "/api/v1/experiments/experiment-1?projectId=project-1", + ); + + expect(response.status).toBe(200); + expect(response.body.currentExecution).toMatchObject({ + requestCount: 3, + requests: [ + { + requestId: "request-active", + runId: "run-active", + status: "processing", + task: "Active task", + criteria: [], + taskPromptId: "prompt-active", + workerType: "coder-acp-copilot", + model: "model-a", + reasoningEffort: "high", + agentVersion: "copilot-dev", + os: { platform: "linux", release: "6.8", arch: "arm64" }, + mcpServers: ["github"], + skillRevisions: ["owner/skill@r1"], + extensions: ["ms-python.python@1.0.0"], + profileId: "profile-a@1", + priority: 5, + attemptNumber: 1, + worker: { instanceId: "worker-1", podName: "pod-1" }, + taxonomyHandler: { + status: "done", + updatedAt: now.toISOString(), + resultAvailable: true, + }, + }, + { + requestId: "request-deleted", + runId: "run-deleted", + status: "deleted", + outcome: "succeeded", + task: "Deleted task", + taxonomyHandler: { + status: "pending", + resultAvailable: false, + }, + }, + { + requestId: "request-missing", + runId: "run-missing", + status: "missing", + }, + ], + }); + expect(requestFind).toHaveBeenCalledTimes(1); + expect(runsFind).toHaveBeenCalledTimes(1); + expect(readExecutionRequests).toHaveBeenCalledWith({ + projectId: "project-1", + experimentId: "experiment-1", + experimentExecutionId: "execution-1", + includeDeleted: false, + }); + + get.mockRestore(); + readExecutionRequests.mockRestore(); + }); + + it("hydrates a historical execution from the exact archived run", async () => { + const now = new Date("2026-10-29T20:00:00.000Z"); + const states = [ + { requestId: "request-1", runId: "run-historical", dependencies: [], blockers: [] }, + { requestId: "request-2", runId: "run-historical-unselected", dependencies: [], blockers: [] }, + ]; + const execution = { + experimentExecutionId: "execution-historical", + attemptNumber: 1, + version: 2, + requests: { + storage: "inline" as const, + entryCount: 1, + payloadFingerprint: "payload", + payloadVersion: 1, + entries: states, + }, + vectorFingerprint: "vector", + handlerSummary: {}, + blocking: [], + createdAt: now, + updatedAt: now, + archivedAt: now, + }; + const getExecution = vi + .spyOn(ExperimentStore.prototype, "getExecution") + .mockResolvedValue(execution); + const readExecutionRequests = vi + .spyOn(ExperimentStore.prototype, "readExecutionRequests") + .mockResolvedValue(states); + const findOne = vi.fn().mockResolvedValue({ + _id: "experiment-1", + experimentId: "experiment-1", + projectId: "project-1", + }); + const requestFind = vi.fn(() => ({ + toArray: vi.fn().mockResolvedValue([ + { + _id: "request-1", + projectId: "project-1", + scenario: { task: "Historical task", criteria: [] }, + workerType: "coder-acp-copilot", + model: "model-a", + run: { + _id: "run-current", + attemptNumber: 2, + status: "queued", + }, + }, + { + _id: "request-2", + projectId: "project-1", + scenario: { task: "Historical unselected task", criteria: [] }, + workerType: "coder-acp-copilot", + model: "model-a", + run: { + _id: "run-current-2", + attemptNumber: 2, + status: "queued", + }, + }, + ]), + })); + const runsFind = vi.fn(() => ({ + toArray: vi.fn().mockResolvedValue([ + { + _id: "run-historical", + requestId: "request-1", + projectId: "project-1", + attemptNumber: 1, + status: "done", + outcome: "failed", + os: { platform: "linux", release: "6.7", arch: "x64" }, + worker: { instanceId: "worker-history" }, + startedAt: now, + handlerStatus: { + "pp-taxonomy": { + status: "processing", + version: 1, + updatedAt: now, + claimToken: "claim-history", + }, + }, + }, + { + _id: "run-historical-unselected", + requestId: "request-2", + projectId: "project-1", + attemptNumber: 1, + status: "done", + outcome: "succeeded", + startedAt: now, + }, + ]), + })); + const { app } = harness({ + experimentCollection: { findOne }, + requestCollection: { find: requestFind }, + runsCollection: { find: runsFind }, + } as unknown as Partial); + + const response = await request(app).get( + "/api/v1/experiments/experiment-1/executions/execution-historical?projectId=project-1", + ); + + expect(response.status).toBe(200); + expect(response.body).toMatchObject({ + experimentExecutionId: "execution-historical", + requestCount: 2, + requests: [ + { + requestId: "request-1", + runId: "run-historical", + status: "done", + outcome: "failed", + task: "Historical task", + workerType: "coder-acp-copilot", + model: "model-a", + attemptNumber: 1, + os: { platform: "linux", release: "6.7", arch: "x64" }, + worker: { instanceId: "worker-history" }, + taxonomyHandler: { + status: "processing", + updatedAt: now.toISOString(), + resultAvailable: false, + }, + }, + { + requestId: "request-2", + runId: "run-historical-unselected", + status: "done", + outcome: "succeeded", + task: "Historical unselected task", + workerType: "coder-acp-copilot", + model: "model-a", + attemptNumber: 1, + }, + ], + }); + expect(response.body.requests[1]).not.toHaveProperty("taxonomyHandler"); + expect(requestFind).toHaveBeenCalledTimes(1); + expect(runsFind).toHaveBeenCalledTimes(1); + + getExecution.mockRestore(); + readExecutionRequests.mockRestore(); + }); + it("lists reports for one exact current or archived execution", async () => { const execution = { experimentExecutionId: "execution-2", diff --git a/apps/api/src/routes/experiments.ts b/apps/api/src/routes/experiments.ts index 7084376c2..edb7eaf94 100644 --- a/apps/api/src/routes/experiments.ts +++ b/apps/api/src/routes/experiments.ts @@ -9,6 +9,7 @@ import type { Response } from "express"; import { CreateRequestInputSchema, ExperimentStore, + MULTI_TURN_DEFAULTS, fingerprintExperimentPayload, ReportResponseSchema, withRetry, @@ -17,6 +18,9 @@ import { type ExperimentExecutionHistoryDocument, type ExperimentMetadata, type ExperimentRequestState, + type ProfileVersionDocument, + parseProfileSpec, + taxonomyHandlerSummary, } from "shared"; import type { RequestDocument, RouteContext } from "../route-context.js"; import { apiRoute } from "../openapi/api-route.js"; @@ -35,6 +39,10 @@ import { ExperimentLifecycleCoordinator, ExperimentLifecycleError, } from "../experiment-lifecycle.js"; +import { + requestedAgentCapabilities, + validateAgentTarget, +} from "../utils/agent-helpers.js"; const CursorSchema = z.string().min(1).optional(); const PageQuerySchema = ProjectIdQuerySchema.extend({ @@ -52,6 +60,7 @@ const ExperimentRequestInputSchema = CreateRequestInputSchema.omit({ profileVariations: true, }).extend({ workerType: z.string().trim().min(1), + agentVersion: z.string().trim().min(1).optional(), }); const CreateExperimentInputSchema = z @@ -61,6 +70,7 @@ const CreateExperimentInputSchema = z requests: z.array(ExperimentRequestInputSchema).max(250_000).default([]), requestIds: z.array(z.string().min(1)).max(250_000).default([]), reportTemplateIds: z.array(z.string().min(1)).max(1_000).default([]), + taxonomyRevisionId: z.string().min(1).optional(), }) .superRefine((value, ctx) => { if ( @@ -121,6 +131,43 @@ const ExperimentReportReferenceSchema = z.object({ updatedAt: z.coerce.date(), }); +const ExperimentRequestSummarySchema = z.object({ + requestId: z.string(), + runId: z.string(), + task: z.string().optional(), + criteria: z.array(z.string()).optional(), + taskPromptId: z.string().optional(), + status: z.string().optional(), + outcome: z.string().optional(), + workerType: z.string().optional(), + model: z.string().optional(), + reasoningEffort: z.string().optional(), + agentVersion: z.string().optional(), + os: z.object({ + platform: z.string(), + release: z.string(), + arch: z.string(), + }).optional(), + mcpServers: z.array(z.string()).optional(), + skillRevisions: z.array(z.string()).optional(), + extensions: z.array(z.string()).optional(), + profileId: z.string().optional(), + priority: z.number().optional(), + attemptNumber: z.number().int().positive().optional(), + worker: z.object({ + instanceId: z.string(), + podName: z.string().optional(), + }).optional(), + lastHeartbeatAt: z.coerce.date().optional(), + startedAt: z.coerce.date().optional(), + taxonomyHandler: z.object({ + status: z.enum(["pending", "queued", "processing", "done", "failed"]), + updatedAt: z.coerce.date().optional(), + error: z.string().optional(), + resultAvailable: z.boolean(), + }).strict().optional(), +}); + const ExperimentTemplateReportSummarySchema = z.object({ templateId: z.string(), templateVersionId: z.string().optional(), @@ -166,12 +213,22 @@ const ExperimentResponseSchema = z revisionNumber: z.number().int(), memberCount: z.number().int(), reportTemplateIds: z.array(z.string()), + taxonomy: z + .object({ + taxonomyRevisionId: z.string(), + ref: z.string(), + contentSha256: z.string(), + }) + .optional(), createdAt: z.coerce.date(), }), currentExecution: z.object({ experimentExecutionId: z.string(), attemptNumber: z.number().int(), + version: z.number().int().optional(), readiness: ReadinessSchema, + requestCount: z.number().int().nonnegative(), + requests: z.array(ExperimentRequestSummarySchema), blockers: z.array( z.object({ requestId: z.string().optional(), @@ -261,7 +318,10 @@ const ExecutionResponseSchema = z .object({ experimentExecutionId: z.string(), attemptNumber: z.number().int(), + version: z.number().int().optional(), readiness: ReadinessSchema, + requestCount: z.number().int().nonnegative(), + requests: z.array(ExperimentRequestSummarySchema), blockers: z.array(z.object({ requestId: z.string().optional(), runId: z.string().optional(), @@ -406,9 +466,12 @@ function sendLifecycleError(res: Response, error: unknown): boolean { return true; } -function membershipResponse(result: ReplaceMembershipResult) { +async function membershipResponse( + ctx: RouteContext, + result: ReplaceMembershipResult, +) { return { - experiment: experimentResponse(result.experiment), + experiment: await resolvedExperimentResponse(ctx, result.experiment), attachedRequestIds: result.attachedRequestIds, detachedRequestIds: result.detachedRequestIds, }; @@ -486,7 +549,10 @@ function executionResponse(execution: ExperimentExecution) { return { experimentExecutionId: execution.experimentExecutionId, attemptNumber: execution.attemptNumber, + version: execution.version, readiness: readiness(execution), + requestCount: execution.requests.entryCount, + requests: [], blockers: execution.blocking, progress: Object.fromEntries( Object.entries(execution.handlerSummary).map(([handlerId, status]) => [ @@ -519,6 +585,152 @@ function executionResponse(execution: ExperimentExecution) { }; } +/** + * Resolves one immutable execution vector into ordered request/run summaries. + * The vector is read once, active request documents are fetched in one query, + * and any non-current exact runs are fetched from history in one query. + */ +async function resolvedExecutionResponse( + ctx: RouteContext, + input: { + projectId: string; + experimentId: string; + execution: ExperimentExecution; + includeDeleted?: boolean; + }, +) { + const states = await store(ctx).readExecutionRequests({ + projectId: input.projectId, + experimentId: input.experimentId, + experimentExecutionId: input.execution.experimentExecutionId, + includeDeleted: input.includeDeleted, + }); + if (!states) return executionResponse(input.execution); + + const requestIds = states.map(({ requestId }) => requestId); + const requestDocuments = requestIds.length === 0 + ? [] + : await ctx.requestCollection + .find( + { + _id: { $in: requestIds }, + projectId: input.projectId, + } as Document, + { + projection: { + _id: 1, + "scenario.task": 1, + "scenario.criteria": 1, + taskPromptId: 1, + workerType: 1, + model: 1, + reasoningEffort: 1, + agentVersion: 1, + mcpServers: 1, + skillRevisions: 1, + extensions: 1, + profileId: 1, + priority: 1, + deletedAt: 1, + run: 1, + }, + }, + ) + .toArray(); + const requestsById = new Map( + requestDocuments.map((request) => [request._id, request]), + ); + + // Exact historical executions may reference a run no longer embedded on its request. + const historicalRunIds = states + .filter(({ requestId, runId }) => { + const request = requestsById.get(requestId); + return request?.run?._id !== runId; + }) + .map(({ runId }) => runId); + const historicalRuns = historicalRunIds.length === 0 + ? [] + : await ctx.runsCollection + .find( + { + _id: { $in: historicalRunIds }, + projectId: input.projectId, + } as Document, + { + projection: { + _id: 1, + status: 1, + outcome: 1, + attemptNumber: 1, + os: 1, + worker: 1, + lastHeartbeatAt: 1, + startedAt: 1, + handlerStatus: 1, + taxonomyEvaluation: 1, + }, + }, + ) + .toArray(); + const historicalRunsById = new Map( + historicalRuns.map((run) => [run._id, run]), + ); + + const requests = states.map(({ requestId, runId }) => { + const request = requestsById.get(requestId); + const run = request?.run?._id === runId + ? request.run + : historicalRunsById.get(runId); + const taxonomyHandler = taxonomyHandlerSummary( + run, + Boolean(input.execution.taxonomy && run), + ); + const status = request?.deletedAt + ? "deleted" + : (run?.status ?? "missing"); + return { + requestId, + runId, + ...(request?.scenario.task ? { task: request.scenario.task } : {}), + ...(request?.scenario.criteria + ? { criteria: request.scenario.criteria } + : {}), + ...(request?.taskPromptId ? { taskPromptId: request.taskPromptId } : {}), + status, + ...(run?.outcome ? { outcome: run.outcome } : {}), + ...(request?.workerType ? { workerType: request.workerType } : {}), + ...(request?.model ? { model: request.model } : {}), + ...(request?.reasoningEffort + ? { reasoningEffort: request.reasoningEffort } + : {}), + ...(request?.agentVersion + ? { agentVersion: request.agentVersion } + : {}), + ...(run?.os ? { os: run.os } : {}), + ...(request?.mcpServers ? { mcpServers: request.mcpServers } : {}), + ...(request?.skillRevisions + ? { skillRevisions: request.skillRevisions } + : {}), + ...(request?.extensions ? { extensions: request.extensions } : {}), + ...(request?.profileId ? { profileId: request.profileId } : {}), + ...(request?.priority !== undefined ? { priority: request.priority } : {}), + ...(run?.attemptNumber ? { attemptNumber: run.attemptNumber } : {}), + ...(run?.worker ? { worker: run.worker } : {}), + ...(run?.lastHeartbeatAt + ? { lastHeartbeatAt: run.lastHeartbeatAt } + : {}), + ...(run?.startedAt ? { startedAt: run.startedAt } : {}), + ...(taxonomyHandler ? { taxonomyHandler } : {}), + }; + }); + + return { + ...executionResponse(input.execution), + requestCount: states.length, + requests, + }; +} + async function streamExperimentExecutionLogs( ctx: RouteContext, req: import("express").Request, @@ -679,6 +891,7 @@ function experimentResponse(experiment: ExperimentDocument) { revisionNumber: experiment.revision.revisionNumber, memberCount: experiment.revision.memberCount, reportTemplateIds: experiment.revision.reportTemplateIds, + taxonomy: experiment.revision.taxonomy, createdAt: experiment.revision.createdAt, }, currentExecution: executionResponse(experiment.execution), @@ -706,6 +919,22 @@ function experimentResponse(experiment: ExperimentDocument) { }; } +/** Returns the detail representation with the exact current execution hydrated. */ +async function resolvedExperimentResponse( + ctx: RouteContext, + experiment: ExperimentDocument, +) { + return { + ...experimentResponse(experiment), + currentExecution: await resolvedExecutionResponse(ctx, { + projectId: experiment.projectId, + experimentId: experiment.experimentId, + execution: experiment.execution, + includeDeleted: Boolean(experiment.deletedAt || experiment.lifecycle), + }), + }; +} + interface ExecutionCursor { attemptNumber: number; experimentExecutionId: string; @@ -779,6 +1008,7 @@ function createRequestDocument(input: { ? { codebaseRevisionId: body.codebaseRevisionId } : {}), ...(body.extensions ? { extensions: body.extensions } : {}), + ...(body.agentVersion ? { agentVersion: body.agentVersion } : {}), ...(body.profileId ? { profileId: body.profileId } : {}), ...(body.gates ? { gates: body.gates } : {}), ...(body.observations ? { observations: body.observations } : {}), @@ -798,6 +1028,82 @@ function createRequestDocument(input: { }; } +async function resolveProfileControlledRequest( + ctx: RouteContext, + projectId: string, + body: z.infer, +): Promise< + | { ok: true; body: z.infer } + | { ok: false; status: 400 | 404; error: string } +> { + if (!body.profileId) return { ok: true, body }; + + let parsed: { profileId: string; version?: number }; + try { + parsed = parseProfileSpec(body.profileId); + } catch (error) { + return { + ok: false, + status: 400, + error: error instanceof Error ? error.message : "Invalid profile reference", + }; + } + const profile = await ctx.profileCollection.findOne({ + _id: parsed.profileId, + projectId, + deletedAt: { $exists: false }, + }); + if (!profile) { + return { ok: false, status: 404, error: `Profile not found: ${parsed.profileId}` }; + } + const version = await ctx.profileVersionCollection.findOne({ + profileId: profile._id, + projectId, + version: parsed.version ?? profile.latestVersion, + }) as ProfileVersionDocument | null; + if (!version) { + return { + ok: false, + status: 404, + error: `Profile version not found: ${body.profileId}`, + }; + } + + // The API owns profile composition: discard client-materialized controlled + // values and independently derive every request from the exact profile. + const { + workerType: _workerType, + model: _model, + reasoningEffort: _reasoningEffort, + agentVersion: _agentVersion, + mcpServers: _mcpServers, + skillRevisions: _skillRevisions, + resources: _resources, + extensions: _extensions, + profileId: _profileId, + ...sharedIntent + } = body; + return { + ok: true, + body: { + ...sharedIntent, + workerType: version.workerType, + model: version.model, + ...(version.reasoningEffort + ? { reasoningEffort: version.reasoningEffort } + : {}), + ...(version.agentVersion ? { agentVersion: version.agentVersion } : {}), + ...(version.mcpServers ? { mcpServers: version.mcpServers } : {}), + ...(version.skillRevisions + ? { skillRevisions: version.skillRevisions } + : {}), + ...(version.resources ? { resources: version.resources } : {}), + ...(version.extensions ? { extensions: version.extensions } : {}), + profileId: version.ref, + }, + }; +} + export function registerExperimentsRoutes(ctx: RouteContext): void { apiRoute(ctx.app, ctx.registry, { method: "post", @@ -820,6 +1126,39 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { const reportTemplateIds = canonicalTemplateIds( req.body.reportTemplateIds, ); + let taxonomy: + | { + taxonomyRevisionId: string; + ref: string; + contentSha256: string; + } + | undefined; + if (req.body.taxonomyRevisionId) { + const enabled = await ctx.featureFlagCollection.findOne({ + key: "taxonome", + enabled: true, + }); + if (!enabled) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const revision = await ctx.taxonomyRevisionStore.get( + req.body.taxonomyRevisionId, + ); + if (!revision) { + res.status(400).json({ error: "Taxonomy revision not found" }); + return; + } + if (revision.projectId !== projectId) { + res.status(400).json({ error: "Taxonomy revision project mismatch" }); + return; + } + taxonomy = { + taxonomyRevisionId: revision._id, + ref: revision.ref, + contentSha256: revision.contentSha256, + }; + } const invalidReportTemplateIds = await invalidManualExperimentTemplateIds( ctx, @@ -833,6 +1172,75 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { }); return; } + const targetCache = new Map< + string, + ReturnType + >(); + const resolvedRequests: Array< + z.infer + > = []; + for (const requestedBody of req.body.requests) { + const effectiveMaxIterations = + requestedBody.maxIterations ?? MULTI_TURN_DEFAULTS.MAX_ITERATIONS; + if ( + effectiveMaxIterations > 1 && + requestedBody.scenario.criteria.length === 0 + ) { + res.status(400).json({ + error: + "At least one criterion is required in scenario.criteria when maxIterations > 1", + }); + return; + } + const profileResolution = await resolveProfileControlledRequest( + ctx, + projectId, + requestedBody, + ); + if (!profileResolution.ok) { + res.status(profileResolution.status).json({ + error: profileResolution.error, + }); + return; + } + const body = profileResolution.body; + const capabilities = requestedAgentCapabilities({ + reasoningEffort: body.reasoningEffort, + mcpServers: body.mcpServers, + skillRevisions: body.skillRevisions, + extensions: body.extensions, + resources: body.resources, + }); + const targetKey = JSON.stringify({ + workerType: body.workerType, + agentVersion: body.agentVersion, + model: body.model, + capabilities, + }); + let targetPromise = targetCache.get(targetKey); + if (!targetPromise) { + targetPromise = validateAgentTarget(ctx.agentCollection, { + workerType: body.workerType, + requestedVersion: body.agentVersion, + model: body.model, + requirements: capabilities, + strictCapabilities: ctx.strictAgentCapabilities, + subjectPlural: "Experiment requests", + }); + targetCache.set(targetKey, targetPromise); + } + const target = await targetPromise; + if (!target.ok) { + const { ok: _ok, status, ...payload } = target; + res.status(status).json(payload); + return; + } + resolvedRequests.push({ + ...body, + agentVersion: target.agentVersion, + ...(target.model ? { model: target.model } : {}), + }); + } const experimentId = uuidv4(); const experimentExecutionId = uuidv4(); const experimentRevisionId = uuidv4(); @@ -847,10 +1255,14 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { metadata: req.body.metadata, requestIds: req.body.requestIds, reportTemplateIds, + taxonomy, now, }); res.status(201).json({ - experiment: experimentResponse(result.experiment), + experiment: await resolvedExperimentResponse( + ctx, + result.experiment, + ), requestIds: result.attachedRequestIds, }); } catch (error) { @@ -858,12 +1270,12 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { } return; } - const identities = req.body.requests.map(() => ({ + const identities = resolvedRequests.map(() => ({ requestId: uuidv4(), runId: uuidv4(), })); const requestDocs = await Promise.all( - req.body.requests.map(async (body, index) => { + resolvedRequests.map(async (body, index) => { const identity = identities[index]!; const taskPrompt = await ctx.taskPromptStore.findOrCreate( projectId, @@ -872,7 +1284,7 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { return createRequestDocument({ ...identity, experimentId, - expectedCount: req.body.requests.length, + expectedCount: resolvedRequests.length, index, projectId, body, @@ -909,9 +1321,11 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { reportTemplateIds, definitionFingerprint: fingerprintExperimentPayload({ metadata, - requests: req.body.requests, + requests: resolvedRequests, reportTemplateIds, + taxonomy, }), + ...(taxonomy ? { taxonomy } : {}), createdAt: now, }, experimentExecutionId, @@ -943,7 +1357,7 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { ); } res.status(201).json({ - experiment: experimentResponse(finalized), + experiment: await resolvedExperimentResponse(ctx, finalized), requestIds: requestDocs.map((doc) => doc._id), }); }, @@ -971,7 +1385,7 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { req.body.expectedExperimentRevisionId, requestIds: req.body.requestIds, }); - res.status(201).json(membershipResponse(result)); + res.status(201).json(await membershipResponse(ctx, result)); } catch (error) { if (!sendMembershipError(res, error)) throw error; } @@ -1000,7 +1414,7 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { req.body.expectedExperimentRevisionId, requestIds: req.body.requestIds, }); - res.status(201).json(membershipResponse(result)); + res.status(201).json(await membershipResponse(ctx, result)); } catch (error) { if (!sendMembershipError(res, error)) throw error; } @@ -1054,7 +1468,7 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { res.status(404).json({ error: "Experiment not found" }); return; } - res.json(experimentResponse(experiment)); + res.json(await resolvedExperimentResponse(ctx, experiment)); }, }); @@ -1244,7 +1658,7 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { }); return; } - res.json(experimentResponse(updated)); + res.json(await resolvedExperimentResponse(ctx, updated)); }, }); @@ -1362,7 +1776,12 @@ export function registerExperimentsRoutes(ctx: RouteContext): void { res.status(404).json({ error: "Experiment execution not found" }); return; } - res.json(executionResponse(execution)); + res.json(await resolvedExecutionResponse(ctx, { + projectId: experiment.projectId, + experimentId: req.params.id, + execution, + includeDeleted: req.query.includeDeleted === "true", + })); }, }); diff --git a/apps/api/src/routes/requests/index.ts b/apps/api/src/routes/requests/index.ts index 231903b0a..2de8d6163 100644 --- a/apps/api/src/routes/requests/index.ts +++ b/apps/api/src/routes/requests/index.ts @@ -38,7 +38,9 @@ import { orderGates, computeTaskPromptId, ExperimentStore, + TaxonomyExpectationStore, LEGACY_REQUEST_COMPOSITION_DEPRECATION_ID, + taxonomyHandlerSummary, } from "shared"; import { trackMetric } from "telemetry"; import type { @@ -1581,7 +1583,13 @@ apiRoute(ctx.app, ctx.registry, { } // Map _id back to id for API response - res.json({ ...resource, id: resource._id }); + res.json({ + ...resource, + id: resource._id, + ...(taxonomyHandlerSummary(resource.run) + ? { taxonomyHandler: taxonomyHandlerSummary(resource.run) } + : {}), + }); }, }); @@ -1941,7 +1949,40 @@ apiRoute(ctx.app, ctx.registry, { resources.reverse(); } - const data = resources.map((r) => ({ ...r, id: r._id })); + const experimentRunPairs = resources.flatMap((resource) => + resource.experimentId && resource.run?._id + ? [{ + experimentId: resource.experimentId, + requestId: resource._id, + runId: resource.run._id, + }] + : [] + ); + const expectedTaxonomyPairs = await new TaxonomyExpectationStore({ + experiments: ctx.experimentCollection, + executions: ctx.experimentExecutionsCollection, + chunks: ctx.experimentChunksCollection, + }).findExpectedPairs(projectId, experimentRunPairs); + const data = resources.map((resource) => { + const exactPair = resource.experimentId && resource.run?._id + ? { + experimentId: resource.experimentId, + requestId: resource._id, + runId: resource.run._id, + } + : undefined; + const taxonomyHandler = taxonomyHandlerSummary( + resource.run, + exactPair + ? expectedTaxonomyPairs.has(TaxonomyExpectationStore.key(exactPair)) + : false, + ); + return { + ...resource, + id: resource._id, + ...(taxonomyHandler ? { taxonomyHandler } : {}), + }; + }); // Enrich `processing` runs with the latest liveness heartbeat from // Redis. The store's mget is cluster-safe (a pipeline of single-key GETs, diff --git a/apps/api/src/routes/requests/taxonomy-evaluation.test.ts b/apps/api/src/routes/requests/taxonomy-evaluation.test.ts new file mode 100644 index 000000000..365be1e9a --- /dev/null +++ b/apps/api/src/routes/requests/taxonomy-evaluation.test.ts @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; +import { gzipSync } from "node:zlib"; +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import supertest from "supertest"; +import { app, _injectTestDependencies } from "../../index.js"; +import { useTestServer } from "../../test-server.js"; +import { createAllMockDependencies } from "../../test-helpers.js"; + +vi.mock("db-migrations/check-migrations", () => ({ + checkMigrations: vi + .fn() + .mockResolvedValue({ ready: true, applied: ["001"], pending: [] }), +})); +vi.mock("../../llm.js", () => ({ + isLlmAvailable: vi.fn().mockReturnValue(false), +})); +vi.mock("../../prompt-feature-llm.js", () => ({ + isLlmAvailable: vi.fn().mockReturnValue(false), +})); +vi.mock("../../task-prompt-llm.js", () => ({ + isTaskPromptLlmAvailable: vi.fn().mockReturnValue(false), +})); + +const validResult = { + run: { + id: "run-1", + scenario: { id: "scenario-1", label: "Scenario" }, + profile: { id: "profile-1", label: "Profile" }, + }, + results: [ + { + schema_version: "scope/taxonomy-evaluation/v1", + status: "succeeded", + taxonomy: { + id: "ax-taxonomy", + version: "1.0.0", + digest: "a".repeat(64), + }, + element: { id: "element-1", label: "Element" }, + implementation: { + type: "deterministic", + handler: "element-1", + attempts: [{ attempt: 1, kind: "initial", outcome: "succeeded" }], + }, + input: { + normalization: "scope/taxonome-normalized-atif/v1", + sha256: "b".repeat(64), + }, + result: { state: "completed", observed: false }, + }, + ], +}; + +function artifactFor(value: unknown) { + const body = Buffer.from(JSON.stringify(value)); + return { + compressed: gzipSync(body), + reference: { + url: "https://storage.example/result.json.gz", + schemaVersion: "2", + contentSha256: createHash("sha256").update(body).digest("hex"), + sizeBytes: body.byteLength, + }, + }; +} + +describe("taxonomy evaluation JSON endpoint", () => { + const testServer = useTestServer(app); + let mocks: ReturnType; + + beforeAll(() => { + mocks = createAllMockDependencies(); + _injectTestDependencies(mocks); + }); + + beforeEach(() => { + vi.clearAllMocks(); + mocks = createAllMockDependencies(); + _injectTestDependencies(mocks); + (mocks.featureFlagCollection.findOne as ReturnType) + .mockResolvedValue({ key: "taxonome", enabled: true }); + }); + + function mockCurrentRun(reference?: ReturnType["reference"]) { + (mocks.collection.findOne as ReturnType).mockResolvedValue({ + _id: "request-1", + run: { + _id: "run-1", + status: "done", + taxonomyEvaluation: reference, + }, + }); + } + + it("returns a validated exact-run result", async () => { + const artifact = artifactFor(validResult); + mockCurrentRun(artifact.reference); + (mocks.blobStorage.downloadBuffer as ReturnType) = vi + .fn() + .mockResolvedValue(artifact.compressed); + + const response = await supertest(testServer()).get( + "/api/v1/requests/request-1/runs/run-1/taxonomy-evaluation/json", + ); + + expect(response.status).toBe(200); + expect(response.body).toEqual(validResult); + }); + + it("fails closed when Taxonome is disabled", async () => { + (mocks.featureFlagCollection.findOne as ReturnType) + .mockResolvedValue(null); + + const response = await supertest(testServer()).get( + "/api/v1/requests/request-1/runs/run-1/taxonomy-evaluation/json", + ); + + expect(response.status).toBe(404); + }); + + it("rejects a run outside the exact request", async () => { + (mocks.collection.findOne as ReturnType).mockResolvedValue({ + _id: "request-1", + run: { _id: "another-run", status: "done" }, + }); + (mocks.runsCollection.findOne as ReturnType) + .mockResolvedValue(null); + + const response = await supertest(testServer()).get( + "/api/v1/requests/request-1/runs/run-1/taxonomy-evaluation/json", + ); + + expect(response.status).toBe(404); + }); + + it("rejects missing and oversized result references before Blob reads", async () => { + mockCurrentRun(); + const missing = await supertest(testServer()).get( + "/api/v1/requests/request-1/runs/run-1/taxonomy-evaluation/json", + ); + expect(missing.status).toBe(404); + + const artifact = artifactFor(validResult); + mockCurrentRun({ + ...artifact.reference, + sizeBytes: 10 * 1024 * 1024 + 1, + }); + const oversized = await supertest(testServer()).get( + "/api/v1/requests/request-1/runs/run-1/taxonomy-evaluation/json", + ); + expect(oversized.status).toBe(413); + }); + + it("reports unavailable and invalid compressed artifacts distinctly", async () => { + const artifact = artifactFor(validResult); + mockCurrentRun(artifact.reference); + (mocks.blobStorage.downloadBuffer as ReturnType) = vi + .fn() + .mockRejectedValue(new Error("missing")); + const unavailable = await supertest(testServer()).get( + "/api/v1/requests/request-1/runs/run-1/taxonomy-evaluation/json", + ); + expect(unavailable.status).toBe(502); + + (mocks.blobStorage.downloadBuffer as ReturnType) = vi + .fn() + .mockResolvedValue(Buffer.from("not gzip")); + const invalid = await supertest(testServer()).get( + "/api/v1/requests/request-1/runs/run-1/taxonomy-evaluation/json", + ); + expect(invalid.status).toBe(409); + }); + + it("rejects size and digest mismatches", async () => { + const artifact = artifactFor(validResult); + mockCurrentRun({ ...artifact.reference, sizeBytes: artifact.reference.sizeBytes + 1 }); + (mocks.blobStorage.downloadBuffer as ReturnType) = vi + .fn() + .mockResolvedValue(artifact.compressed); + const sizeMismatch = await supertest(testServer()).get( + "/api/v1/requests/request-1/runs/run-1/taxonomy-evaluation/json", + ); + expect(sizeMismatch.status).toBe(409); + + mockCurrentRun({ ...artifact.reference, contentSha256: "0".repeat(64) }); + const digestMismatch = await supertest(testServer()).get( + "/api/v1/requests/request-1/runs/run-1/taxonomy-evaluation/json", + ); + expect(digestMismatch.status).toBe(409); + }); + + it("rejects malformed JSON and schema-invalid documents", async () => { + for (const value of ["not json", { run: {}, results: [] }]) { + const body = + typeof value === "string" + ? Buffer.from(value) + : Buffer.from(JSON.stringify(value)); + mockCurrentRun({ + url: "https://storage.example/result.json.gz", + schemaVersion: "2", + contentSha256: createHash("sha256").update(body).digest("hex"), + sizeBytes: body.byteLength, + }); + (mocks.blobStorage.downloadBuffer as ReturnType) = vi + .fn() + .mockResolvedValue(gzipSync(body)); + + const response = await supertest(testServer()).get( + "/api/v1/requests/request-1/runs/run-1/taxonomy-evaluation/json", + ); + expect(response.status).toBe(409); + } + }); +}); diff --git a/apps/api/src/routes/requests/taxonomy-evaluation.ts b/apps/api/src/routes/requests/taxonomy-evaluation.ts new file mode 100644 index 000000000..4baf49f50 --- /dev/null +++ b/apps/api/src/routes/requests/taxonomy-evaluation.ts @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; +import { gunzipSync } from "node:zlib"; +import { z } from "zod"; +import { TaxonomyRunResultResponseSchema } from "shared"; +import { apiRoute } from "../../openapi/api-route.js"; +import type { RouteContext } from "../../route-context.js"; +import { resolveRunForRequest } from "./resolve-run.js"; + +const TaxonomyEvaluationReferenceSchema = z + .object({ + url: z.string(), + schemaVersion: z.string(), + contentSha256: z.string(), + sizeBytes: z.number().int().nonnegative(), + }) + .openapi("TaxonomyEvaluationReference"); + +const configuredMaxBytes = Number(process.env.TAXONOMY_RESULT_MAX_BYTES); +const MAX_TAXONOMY_RESULT_BYTES = + Number.isFinite(configuredMaxBytes) && configuredMaxBytes > 0 + ? Math.floor(configuredMaxBytes) + : 10 * 1024 * 1024; + +async function enabled(ctx: RouteContext): Promise { + return Boolean( + await ctx.featureFlagCollection.findOne({ + key: "taxonome", + enabled: true, + }), + ); +} + +/** Registers exact-run Taxonome result metadata and stable blob proxy routes. */ +export function registerRequestsTaxonomyEvaluationRoutes( + ctx: RouteContext, +): void { + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/requests/:id/runs/:runId/taxonomy-evaluation", + tags: ["Requests"], + summary: "Get the exact-run Taxonome result reference", + response: TaxonomyEvaluationReferenceSchema, + handler: async (req, res, next) => { + try { + if (!(await enabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const resolved = await resolveRunForRequest( + ctx, + req.params.id, + req.params.runId, + ); + if ("error" in resolved) { + res.status(resolved.status).json({ error: resolved.error }); + return; + } + if (!resolved.run.taxonomyEvaluation) { + res.status(404).json({ error: "Taxonomy evaluation not available" }); + return; + } + res.json(resolved.run.taxonomyEvaluation); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/requests/:id/runs/:runId/taxonomy-evaluation/json", + tags: ["Requests"], + summary: "Get the validated exact-run Taxonome result JSON", + response: TaxonomyRunResultResponseSchema, + handler: async (req, res, next) => { + try { + if (!(await enabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const resolved = await resolveRunForRequest( + ctx, + req.params.id, + req.params.runId, + ); + if ("error" in resolved) { + res.status(resolved.status).json({ error: resolved.error }); + return; + } + const reference = resolved.run.taxonomyEvaluation; + if (!reference) { + res.status(404).json({ error: "Taxonomy evaluation not available" }); + return; + } + if (reference.sizeBytes > MAX_TAXONOMY_RESULT_BYTES) { + res.status(413).json({ error: "Taxonomy evaluation exceeds the view limit" }); + return; + } + + let compressed: Buffer; + try { + compressed = await ctx.blobStorage.downloadBuffer(reference.url); + } catch { + res.status(502).json({ error: "Taxonomy evaluation artifact is unavailable" }); + return; + } + + let body: Buffer; + try { + body = gunzipSync(compressed, { + maxOutputLength: MAX_TAXONOMY_RESULT_BYTES, + }); + } catch { + res.status(409).json({ error: "Taxonomy evaluation artifact is invalid" }); + return; + } + if (body.byteLength !== reference.sizeBytes) { + res.status(409).json({ error: "Taxonomy evaluation size check failed" }); + return; + } + const hash = createHash("sha256").update(body).digest("hex"); + if (hash !== reference.contentSha256) { + res.status(409).json({ error: "Taxonomy evaluation integrity check failed" }); + return; + } + + let parsed: unknown; + try { + parsed = JSON.parse(body.toString("utf8")); + } catch { + res.status(409).json({ error: "Taxonomy evaluation JSON is invalid" }); + return; + } + const result = TaxonomyRunResultResponseSchema.safeParse(parsed); + if (!result.success) { + res.status(409).json({ error: "Taxonomy evaluation schema is invalid" }); + return; + } + res.json(result.data); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/requests/:id/runs/:runId/taxonomy-evaluation/content", + tags: ["Requests"], + summary: "Download the exact-run compressed Taxonome result", + response: z.any(), + rawResponse: true, + handler: async (req, res, next) => { + try { + if (!(await enabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const resolved = await resolveRunForRequest( + ctx, + req.params.id, + req.params.runId, + ); + if ("error" in resolved) { + res.status(resolved.status).json({ error: resolved.error }); + return; + } + const reference = resolved.run.taxonomyEvaluation; + if (!reference) { + res.status(404).json({ error: "Taxonomy evaluation not available" }); + return; + } + const body = await ctx.blobStorage.downloadBuffer(reference.url); + res + .type("application/json") + .set("Content-Encoding", "gzip") + .set( + "Content-Disposition", + `attachment; filename="taxonomy-${req.params.runId}.json.gz"`, + ) + .send(body); + } catch (error) { + next(error); + } + }, + }); +} diff --git a/apps/api/src/routes/taxonomies.test.ts b/apps/api/src/routes/taxonomies.test.ts new file mode 100644 index 000000000..321fe3a37 --- /dev/null +++ b/apps/api/src/routes/taxonomies.test.ts @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it, vi } from "vitest"; +import { + listTaxonomyRows, + resolveTaxonomyDetail, + type TaxonomyRouteStores, +} from "./taxonomies.js"; + +const createdAt = new Date("2026-10-29T00:00:00Z"); + +const taxonomyRoot = { + _id: "taxonomy-root", + projectId: "project-a", + id: "example", + name: "Example taxonomy", + sourceType: "user" as const, + revisionCounter: 2, + latestRevisionId: "taxonomy-revision-2", + latestRevision: 2, + createdAt, +}; + +const revision1 = { + _id: "taxonomy-revision-1", + taxonomyId: taxonomyRoot._id, + projectId: taxonomyRoot.projectId, + sourceType: "user" as const, + schema_version: "scope/taxonomy/v1" as const, + id: taxonomyRoot.id, + version: "1.0.0", + surfaces: [], + elements: [{ ref: "element-b@r1" }, { ref: "element-a@r1" }], + revision: 1, + ref: "example@r1", + contentSha256: "1".repeat(64), + createdAt, +}; + +const revision2 = { + ...revision1, + _id: "taxonomy-revision-2", + version: "1.1.0", + elements: [{ ref: "element-a@r2" }], + revision: 2, + ref: "example@r2", + contentSha256: "2".repeat(64), +}; + +function elementRevision( + rootId: string, + id: string, + ref: string, + revision: number, +) { + return { + _id: `${rootId}-revision-${revision}`, + elementId: rootId, + projectId: taxonomyRoot.projectId, + sourceType: "user" as const, + id, + label: id.toUpperCase(), + implementation: { type: "prompt" as const, prompt: `Prompt ${id}` }, + preconditions: {}, + output: { evidence_roles: {} }, + revision, + ref, + contentSha256: String(revision).repeat(64), + createdAt, + }; +} + +function elementRoot(rootId: string, id: string) { + return { + _id: rootId, + projectId: taxonomyRoot.projectId, + id, + name: id.toUpperCase(), + sourceType: "user" as const, + revisionCounter: 2, + latestRevision: 2, + createdAt, + }; +} + +describe("taxonomy detail projection", () => { + it("resolves one historical taxonomy revision with exact ordered elements", async () => { + const elementA = elementRevision("root-a", "element-a", "element-a@r1", 1); + const elementB = elementRevision("root-b", "element-b", "element-b@r1", 1); + const stores = { + taxonomyStore: { + get: vi.fn(async () => taxonomyRoot), + }, + taxonomyRevisionStore: { + list: vi.fn(async () => [revision2, revision1]), + }, + taxonomyElementRevisionStore: { + getByRefs: vi.fn(async () => [elementA, elementB]), + }, + taxonomyElementStore: { + getMany: vi.fn(async () => [ + elementRoot("root-a", "element-a"), + elementRoot("root-b", "element-b"), + ]), + }, + } as unknown as TaxonomyRouteStores; + + const detail = await resolveTaxonomyDetail( + stores, + taxonomyRoot.projectId, + taxonomyRoot._id, + revision1._id, + ); + + expect(detail?.selectedRevision.ref).toBe("example@r1"); + expect(detail?.elements.map(element => element.revision.ref)).toEqual([ + "element-b@r1", + "element-a@r1", + ]); + expect(detail?.elements.map(element => element.position)).toEqual([0, 1]); + expect(detail?.revisions).toEqual([ + expect.objectContaining({ ref: "example@r2", isLatest: true }), + expect.objectContaining({ ref: "example@r1", isLatest: false }), + ]); + expect(stores.taxonomyElementRevisionStore.getByRefs) + .toHaveBeenCalledOnce(); + expect(stores.taxonomyElementStore.getMany).toHaveBeenCalledOnce(); + }); + + it("rejects a root outside the requested project before revision reads", async () => { + const stores = { + taxonomyStore: { get: vi.fn(async () => taxonomyRoot) }, + taxonomyRevisionStore: { list: vi.fn() }, + } as unknown as TaxonomyRouteStores; + + await expect(resolveTaxonomyDetail( + stores, + "project-b", + taxonomyRoot._id, + )).resolves.toBeNull(); + expect(stores.taxonomyRevisionStore.list).not.toHaveBeenCalled(); + }); + + it("enriches taxonomy rows with one bulk latest-revision read", async () => { + const stores = { + taxonomyStore: { list: vi.fn(async () => [taxonomyRoot]) }, + taxonomyRevisionStore: { getMany: vi.fn(async () => [revision2]) }, + } as unknown as TaxonomyRouteStores; + + await expect(listTaxonomyRows(stores, taxonomyRoot.projectId)).resolves + .toEqual([expect.objectContaining({ + _id: taxonomyRoot._id, + ref: "example@r2", + version: "1.1.0", + elementCount: 1, + })]); + expect(stores.taxonomyRevisionStore.getMany).toHaveBeenCalledWith([ + revision2._id, + ]); + }); +}); diff --git a/apps/api/src/routes/taxonomies.ts b/apps/api/src/routes/taxonomies.ts new file mode 100644 index 000000000..84ac236e5 --- /dev/null +++ b/apps/api/src/routes/taxonomies.ts @@ -0,0 +1,831 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { z } from "zod"; +import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; +import { + CopyBuiltinTaxonomyRevisionInputSchema, + CreateTaxonomyElementInputSchema, + CreateTaxonomyElementRevisionInputSchema, + CreateTaxonomyInputSchema, + CreateTaxonomyRevisionInputSchema, + PortableTaxonomyElementSchema, + PortableTaxonomySchema, + TaxonomyElementResponseSchema, + TaxonomyElementRevisionResponseSchema, + TaxonomyDetailResponseSchema, + TaxonomyListItemResponseSchema, + TaxonomyResponseSchema, + TaxonomyRevisionResponseSchema, + UpdateTaxonomyMetadataInputSchema, + type PortableTaxonomyDocument, + type TaxonomyDocument, + type TaxonomyElementDocument, + type TaxonomyDetail, + type TaxonomyListItem, +} from "shared"; +import { apiRoute } from "../openapi/api-route.js"; +import type { RouteContext } from "../route-context.js"; +import { + ProjectIdQuerySchema, + getQueryProjectId, +} from "../utils/project-scope.js"; +import { ensureBuiltinTaxonomy } from "../taxonomy-builtin-seeder.js"; + +const ImportTaxonomyYamlInputSchema = z + .object({ + name: z.string().trim().min(1).max(256), + description: z.string().max(4096).optional(), + creator: z.string().optional(), + taxonomyYaml: z.string().min(1), + elements: z.record(z.string(), z.string()).default({}), + }) + .strict() + .openapi("ImportTaxonomyYamlInput"); + +const TaxonomyDetailQuerySchema = ProjectIdQuerySchema.extend({ + revisionId: z.string().optional(), +}).openapi("TaxonomyDetailQuery"); + +export type TaxonomyRouteStores = Pick< + RouteContext, + | "taxonomyStore" + | "taxonomyRevisionStore" + | "taxonomyElementStore" + | "taxonomyElementRevisionStore" +>; + +/** Taxonome entry points are hidden unless the operator explicitly enables them. */ +async function requireTaxonome(ctx: RouteContext, response: { + status(code: number): { json(body: unknown): void }; +}): Promise { + const flag = await ctx.featureFlagCollection.findOne({ + key: "taxonome", + enabled: true, + }); + if (!flag) { + response.status(404).json({ error: "Taxonome is not enabled" }); + return false; + } + return true; +} + +function assertUserRoot( + root: TaxonomyDocument | TaxonomyElementDocument, +): void { + if (root.sourceType !== "user") { + throw new Error("Built-in taxonomy content is read-only"); + } +} + +function portableTaxonomyRevision(revision: { + schema_version: "scope/taxonomy/v1"; + id: string; + version: string; + surfaces: PortableTaxonomyDocument["surfaces"]; + elements: PortableTaxonomyDocument["elements"]; +}): PortableTaxonomyDocument { + return { + schema_version: revision.schema_version, + id: revision.id, + version: revision.version, + surfaces: revision.surfaces, + elements: revision.elements, + }; +} + +async function validateResolvedTaxonomyElements( + ctx: RouteContext, + projectId: string, + content: PortableTaxonomyDocument, +): Promise { + const elementIds = new Set(); + for (const reference of content.elements) { + if ("path" in reference) { + return `Unresolved element path '${reference.path}'; use the YAML import endpoint`; + } + const revision = await ctx.taxonomyElementRevisionStore.getByRef( + projectId, + reference.ref, + ); + if (!revision) { + return `Element revision '${reference.ref}' was not found in this project`; + } + if (elementIds.has(revision.id)) { + return `Duplicate element id '${revision.id}'`; + } + elementIds.add(revision.id); + } + return undefined; +} + +/** Builds list rows with one bulk latest-revision read, avoiding per-row lookups. */ +export async function listTaxonomyRows( + ctx: TaxonomyRouteStores, + projectId: string, +): Promise { + const roots = await ctx.taxonomyStore.list(projectId); + const revisions = await ctx.taxonomyRevisionStore.getMany( + roots.flatMap(root => root.latestRevisionId ? [root.latestRevisionId] : []), + ); + const revisionsById = new Map(revisions.map(revision => [ + revision._id, + revision, + ])); + return roots.map(root => { + const revision = root.latestRevisionId + ? revisionsById.get(root.latestRevisionId) + : undefined; + if (!revision) { + throw new Error(`Taxonomy root '${root._id}' has no latest revision`); + } + return { + ...root, + ref: revision.ref, + version: revision.version, + elementCount: revision.elements.length, + }; + }); +} + +/** + * Resolves one taxonomy revision and its exact ordered element revision vector + * using bulk reads scoped to the taxonomy's project. + */ +export async function resolveTaxonomyDetail( + ctx: TaxonomyRouteStores, + projectId: string, + taxonomyId: string, + revisionId?: string, +): Promise { + const taxonomy = await ctx.taxonomyStore.get(taxonomyId, true); + if (!taxonomy || taxonomy.projectId !== projectId) return null; + + const revisions = await ctx.taxonomyRevisionStore.list(taxonomy._id); + const selectedRevision = revisionId + ? revisions.find(revision => + revision._id === revisionId || + revision.ref === revisionId || + String(revision.revision) === revisionId + ) + : revisions.find(revision => + revision._id === taxonomy.latestRevisionId + ) ?? revisions[0]; + if (!selectedRevision) return null; + + const refs = selectedRevision.elements.map(reference => { + if (!("ref" in reference)) { + throw new Error( + `Stored taxonomy revision '${selectedRevision.ref}' contains unresolved element paths`, + ); + } + return reference.ref; + }); + const elementRevisions = await ctx.taxonomyElementRevisionStore.getByRefs( + projectId, + refs, + ); + const revisionsByRef = new Map(elementRevisions.map(revision => [ + revision.ref, + revision, + ])); + const elementRoots = await ctx.taxonomyElementStore.getMany( + elementRevisions.map(revision => revision.elementId), + true, + ); + const rootsById = new Map(elementRoots.map(root => [root._id, root])); + const elements = refs.map((ref, position) => { + const revision = revisionsByRef.get(ref); + if (!revision) { + throw new Error( + `Element revision '${ref}' referenced by '${selectedRevision.ref}' was not found`, + ); + } + const root = rootsById.get(revision.elementId); + if (!root || root.projectId !== projectId) { + throw new Error(`Element root for '${ref}' was not found in this project`); + } + return { position, root, revision }; + }); + + return { + taxonomy, + selectedRevision, + revisions: revisions.map(revision => ({ + _id: revision._id, + revision: revision.revision, + ref: revision.ref, + version: revision.version, + elementCount: revision.elements.length, + contentSha256: revision.contentSha256, + ...(revision.creator ? { creator: revision.creator } : {}), + createdAt: revision.createdAt, + isLatest: revision._id === taxonomy.latestRevisionId, + })), + elements, + }; +} + +/** Registers the complete project-scoped taxonomy and element product API. */ +export function registerTaxonomyRoutes(ctx: RouteContext): void { + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomy-elements", + tags: ["Taxonomies"], + summary: "List project taxonomy elements", + query: ProjectIdQuerySchema, + response: z.array(TaxonomyElementResponseSchema), + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + await ensureBuiltinTaxonomy(ctx, getQueryProjectId(req)); + res.json( + await ctx.taxonomyElementStore.list(getQueryProjectId(req)), + ); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/taxonomy-elements", + tags: ["Taxonomies"], + summary: "Create a user taxonomy element and revision 1", + query: ProjectIdQuerySchema, + body: CreateTaxonomyElementInputSchema, + response: TaxonomyElementResponseSchema, + successStatus: 201, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + if (req.body.sourceType !== "user") { + res.status(400).json({ error: "Only user taxonomy elements can be created" }); + return; + } + if (req.body.id && req.body.id !== req.body.content.id) { + res.status(400).json({ error: "Root id must match content id" }); + return; + } + const root = await ctx.taxonomyElementStore.create({ + projectId: getQueryProjectId(req), + id: req.body.content.id, + name: req.body.name, + ...(req.body.description + ? { description: req.body.description } + : {}), + sourceType: "user", + ...(req.body.creator ? { creator: req.body.creator } : {}), + }); + await ctx.taxonomyElementRevisionStore.create( + root._id, + req.body.content, + req.body.creator, + ); + res + .status(201) + .json((await ctx.taxonomyElementStore.get(root._id)) ?? root); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomies", + tags: ["Taxonomies"], + summary: "List project taxonomies", + query: ProjectIdQuerySchema, + response: z.array(TaxonomyListItemResponseSchema), + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + await ensureBuiltinTaxonomy(ctx, getQueryProjectId(req)); + res.json(await listTaxonomyRows(ctx, getQueryProjectId(req))); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomies/:id/detail", + tags: ["Taxonomies"], + summary: "Get taxonomy detail with exact ordered element revisions", + query: TaxonomyDetailQuerySchema, + response: TaxonomyDetailResponseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const detail = await resolveTaxonomyDetail( + ctx, + getQueryProjectId(req), + req.params.id, + req.query.revisionId, + ); + if (!detail) { + res.status(404).json({ error: "Taxonomy or revision not found" }); + return; + } + res.json(detail); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomy-element-revisions/:revisionId/export", + tags: ["Taxonomies"], + summary: "Export an exact taxonomy element revision as portable YAML", + response: z.string(), + rawResponse: true, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const revision = await ctx.taxonomyElementRevisionStore.get( + req.params.revisionId, + ); + if (!revision) { + res.status(404).json({ error: "Taxonomy element revision not found" }); + return; + } + const { + _id: _id, + elementId: _elementId, + projectId: _projectId, + sourceType: _sourceType, + revision: _revision, + ref: _ref, + contentSha256: _contentSha256, + creator: _creator, + createdAt: _createdAt, + ...portable + } = revision; + res.type("application/yaml").send(stringifyYaml(portable)); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/taxonomies", + tags: ["Taxonomies"], + summary: "Create a user taxonomy and revision 1", + query: ProjectIdQuerySchema, + body: CreateTaxonomyInputSchema, + response: TaxonomyResponseSchema, + successStatus: 201, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + if (req.body.sourceType !== "user") { + res.status(400).json({ error: "Only user taxonomies can be created" }); + return; + } + if (req.body.id && req.body.id !== req.body.content.id) { + res.status(400).json({ error: "Root id must match content id" }); + return; + } + const elementError = await validateResolvedTaxonomyElements( + ctx, + getQueryProjectId(req), + req.body.content, + ); + if (elementError) { + res.status(400).json({ error: elementError }); + return; + } + const root = await ctx.taxonomyStore.create({ + projectId: getQueryProjectId(req), + id: req.body.content.id, + name: req.body.name, + ...(req.body.description + ? { description: req.body.description } + : {}), + sourceType: "user", + ...(req.body.creator ? { creator: req.body.creator } : {}), + }); + await ctx.taxonomyRevisionStore.create( + root._id, + req.body.content, + req.body.creator, + ); + res.status(201).json((await ctx.taxonomyStore.get(root._id)) ?? root); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/taxonomies/import", + tags: ["Taxonomies"], + summary: "Import taxonomy YAML and referenced element YAML", + query: ProjectIdQuerySchema, + body: ImportTaxonomyYamlInputSchema, + response: TaxonomyResponseSchema, + successStatus: 201, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const taxonomy = PortableTaxonomySchema.parse( + parseYaml(req.body.taxonomyYaml), + ); + const projectId = getQueryProjectId(req); + const refs: Array<{ ref: string }> = []; + const ids = new Set(); + + for (const reference of taxonomy.elements) { + if ("ref" in reference) { + const resolved = await ctx.taxonomyElementRevisionStore.getByRef( + projectId, + reference.ref, + ); + if (!resolved) { + res.status(400).json({ + error: `Element revision '${reference.ref}' was not found in this project`, + }); + return; + } + if (ids.has(resolved.id)) { + res.status(400).json({ error: `Duplicate element id '${resolved.id}'` }); + return; + } + ids.add(resolved.id); + refs.push({ ref: reference.ref }); + continue; + } + + const yaml = req.body.elements[reference.path]; + if (!yaml) { + res.status(400).json({ + error: `Missing element file '${reference.path}'`, + }); + return; + } + const content = PortableTaxonomyElementSchema.parse(parseYaml(yaml)); + if (ids.has(content.id)) { + res.status(400).json({ error: `Duplicate element id '${content.id}'` }); + return; + } + ids.add(content.id); + const element = await ctx.taxonomyElementStore.create({ + projectId, + id: content.id, + name: content.label, + ...(content.description ? { description: content.description } : {}), + sourceType: "user", + ...(req.body.creator ? { creator: req.body.creator } : {}), + }); + const created = await ctx.taxonomyElementRevisionStore.create( + element._id, + content, + req.body.creator, + ); + refs.push({ ref: created.revision.ref }); + } + + const root = await ctx.taxonomyStore.create({ + projectId, + id: taxonomy.id, + name: req.body.name, + ...(req.body.description + ? { description: req.body.description } + : {}), + sourceType: "user", + ...(req.body.creator ? { creator: req.body.creator } : {}), + }); + await ctx.taxonomyRevisionStore.create( + root._id, + { ...taxonomy, elements: refs }, + req.body.creator, + ); + res.status(201).json((await ctx.taxonomyStore.get(root._id)) ?? root); + } catch (error) { + next(error); + } + }, + }); + + for (const operation of [ + { path: "/api/v1/taxonomies/:id", element: false }, + { path: "/api/v1/taxonomy-elements/:id", element: true }, + ] as const) { + const responseSchema = operation.element + ? TaxonomyElementResponseSchema + : TaxonomyResponseSchema; + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: operation.path, + tags: ["Taxonomies"], + summary: `Get a taxonomy${operation.element ? " element" : ""}`, + response: responseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const rootStore = operation.element + ? ctx.taxonomyElementStore + : ctx.taxonomyStore; + const root = await rootStore.get(req.params.id); + if (!root) { + res.status(404).json({ error: "Taxonomy root not found" }); + return; + } + res.json(root); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "patch", + path: operation.path, + tags: ["Taxonomies"], + summary: `Update taxonomy${operation.element ? " element" : ""} metadata`, + body: UpdateTaxonomyMetadataInputSchema, + response: responseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const rootStore = operation.element + ? ctx.taxonomyElementStore + : ctx.taxonomyStore; + const root = await rootStore.get(req.params.id); + if (!root) { + res.status(404).json({ error: "Taxonomy root not found" }); + return; + } + assertUserRoot(root); + res.json(await rootStore.updateMetadata(req.params.id, req.body)); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "delete", + path: operation.path, + tags: ["Taxonomies"], + summary: `Soft-delete a taxonomy${operation.element ? " element" : ""}`, + response: z.any(), + rawResponse: true, + successStatus: 204, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const rootStore = operation.element + ? ctx.taxonomyElementStore + : ctx.taxonomyStore; + if (!(await rootStore.softDelete(req.params.id))) { + res.status(404).json({ error: "User taxonomy root not found" }); + return; + } + res.status(204).send(); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: `${operation.path}/restore`, + tags: ["Taxonomies"], + summary: `Restore a taxonomy${operation.element ? " element" : ""}`, + response: responseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const rootStore = operation.element + ? ctx.taxonomyElementStore + : ctx.taxonomyStore; + const restored = await rootStore.restore(req.params.id); + if (!restored) { + res.status(404).json({ error: "Deleted user taxonomy root not found" }); + return; + } + res.json(restored); + } catch (error) { + next(error); + } + }, + }); + } + + for (const operation of [ + { path: "/api/v1/taxonomies/:id/revisions", element: false }, + { path: "/api/v1/taxonomy-elements/:id/revisions", element: true }, + ] as const) { + const bodySchema = operation.element + ? CreateTaxonomyElementRevisionInputSchema + : CreateTaxonomyRevisionInputSchema; + const responseSchema = operation.element + ? TaxonomyElementRevisionResponseSchema + : TaxonomyRevisionResponseSchema; + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: operation.path, + tags: ["Taxonomies"], + summary: `List taxonomy${operation.element ? " element" : ""} revisions`, + response: z.array(responseSchema), + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const rootStore = operation.element + ? ctx.taxonomyElementStore + : ctx.taxonomyStore; + const revisionStore = operation.element + ? ctx.taxonomyElementRevisionStore + : ctx.taxonomyRevisionStore; + const root = await rootStore.get(req.params.id, true); + if (!root) { + res.status(404).json({ error: "Taxonomy root not found" }); + return; + } + res.json(await revisionStore.list(root._id)); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: operation.path, + tags: ["Taxonomies"], + summary: `Create taxonomy${operation.element ? " element" : ""} revision`, + body: bodySchema, + response: responseSchema, + successStatus: 201, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const rootStore = operation.element + ? ctx.taxonomyElementStore + : ctx.taxonomyStore; + const revisionStore = operation.element + ? ctx.taxonomyElementRevisionStore + : ctx.taxonomyRevisionStore; + const root = await rootStore.get(req.params.id); + if (!root) { + res.status(404).json({ error: "Taxonomy root not found" }); + return; + } + assertUserRoot(root); + if (req.body.content.id !== root.id) { + res.status(400).json({ error: "Revision id must match root id" }); + return; + } + if (!operation.element) { + const elementError = await validateResolvedTaxonomyElements( + ctx, + root.projectId, + req.body.content as PortableTaxonomyDocument, + ); + if (elementError) { + res.status(400).json({ error: elementError }); + return; + } + } + const result = await revisionStore.create( + root._id, + req.body.content as never, + req.body.creator, + ); + res.status(result.deduplicated ? 200 : 201).json(result.revision); + } catch (error) { + next(error); + } + }, + }); + } + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomy-revisions/:revisionId", + tags: ["Taxonomies"], + summary: "Get an exact taxonomy revision", + response: TaxonomyRevisionResponseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const revision = await ctx.taxonomyRevisionStore.get( + req.params.revisionId, + ); + if (!revision) { + res.status(404).json({ error: "Taxonomy revision not found" }); + return; + } + res.json(revision); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomy-element-revisions/:revisionId", + tags: ["Taxonomies"], + summary: "Get an exact taxonomy element revision", + response: TaxonomyElementRevisionResponseSchema, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const revision = await ctx.taxonomyElementRevisionStore.get( + req.params.revisionId, + ); + if (!revision) { + res.status(404).json({ error: "Taxonomy element revision not found" }); + return; + } + res.json(revision); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/taxonomy-revisions/:revisionId/export", + tags: ["Taxonomies"], + summary: "Export an exact taxonomy revision as portable YAML", + response: z.string(), + rawResponse: true, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const revision = await ctx.taxonomyRevisionStore.get( + req.params.revisionId, + ); + if (!revision) { + res.status(404).json({ error: "Taxonomy revision not found" }); + return; + } + res.type("application/yaml").send( + stringifyYaml(portableTaxonomyRevision(revision)), + ); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/taxonomy-revisions/:revisionId/copy", + tags: ["Taxonomies"], + summary: "Copy an exact built-in revision to user revision 1", + query: ProjectIdQuerySchema, + body: CopyBuiltinTaxonomyRevisionInputSchema, + response: TaxonomyResponseSchema, + successStatus: 201, + handler: async (req, res, next) => { + try { + if (!(await requireTaxonome(ctx, res))) return; + const source = await ctx.taxonomyRevisionStore.get(req.params.revisionId); + if (!source || source.sourceType !== "builtin") { + res.status(404).json({ error: "Built-in taxonomy revision not found" }); + return; + } + if (source.projectId !== getQueryProjectId(req)) { + res.status(400).json({ error: "Taxonomy revision project mismatch" }); + return; + } + const content = portableTaxonomyRevision(source); + const id = req.body.id ?? `${source.id}-copy`; + const root = await ctx.taxonomyStore.create({ + projectId: source.projectId, + id, + name: req.body.name, + ...(req.body.description + ? { description: req.body.description } + : {}), + sourceType: "user", + ...(req.body.creator ? { creator: req.body.creator } : {}), + }); + await ctx.taxonomyRevisionStore.create( + root._id, + { ...content, id }, + req.body.creator, + ); + res.status(201).json((await ctx.taxonomyStore.get(root._id)) ?? root); + } catch (error) { + next(error); + } + }, + }); +} diff --git a/apps/api/src/routes/taxonomy-readout.test.ts b/apps/api/src/routes/taxonomy-readout.test.ts new file mode 100644 index 000000000..2f76d1823 --- /dev/null +++ b/apps/api/src/routes/taxonomy-readout.test.ts @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; +import { gzipSync } from "node:zlib"; +import { describe, expect, it, vi } from "vitest"; +import { buildTaxonomyReadout } from "./taxonomy-readout.js"; + +describe("buildTaxonomyReadout", () => { + it("preserves all three exact request/run identities in one readout", async () => { + const pairs = [1, 2, 3].map((number) => ({ + requestId: `request-${number}`, + runId: `run-${number}`, + dependencies: [], + blockers: [], + })); + const bodies = new Map( + pairs.map((pair) => { + const body = Buffer.from( + JSON.stringify({ + run: { + id: pair.runId, + scenario: { id: "scenario", label: "Scenario" }, + profile: { id: "profile", label: "Profile" }, + }, + results: [], + }), + "utf8", + ); + return [pair.runId, body] as const; + }), + ); + const requestCollection = { + findOne: vi.fn(async ({ _id }: { _id: string }) => { + const pair = pairs.find((candidate) => candidate.requestId === _id)!; + const body = bodies.get(pair.runId)!; + return { + _id, + projectId: "project-1", + scenario: { task: "Task" }, + run: { + _id: pair.runId, + attemptNumber: 1, + status: "done", + handlerStatus: { + "pp-taxonomy": { + status: "done", + version: 1, + updatedAt: new Date(), + }, + }, + taxonomyEvaluation: { + url: `https://blob/${pair.runId}.json.gz`, + schemaVersion: "scope/taxonomy-run-result/v1", + contentSha256: createHash("sha256").update(body).digest("hex"), + sizeBytes: body.byteLength, + }, + }, + }; + }), + }; + const blobStorage = { + downloadBuffer: vi.fn(async (url: string) => { + const runId = url.match(/(run-\d+)/)?.[1]; + return gzipSync(bodies.get(runId!)!); + }), + }; + + const readout = await buildTaxonomyReadout( + { + requestCollection, + runsCollection: { findOne: vi.fn() }, + blobStorage, + } as never, + { + projectId: "project-1", + experimentId: "experiment-1", + executionId: "execution-1", + requests: pairs, + taxonomy: { + taxonomyRevisionId: "taxonomy-revision-1", + ref: "ax-taxonomy@r1", + contentSha256: "a".repeat(64), + }, + }, + ); + + expect(readout.runs).toHaveLength(3); + expect( + readout.runs.map((run) => [run.requestId, run.runId, run.state]), + ).toEqual([ + ["request-1", "run-1", "completed"], + ["request-2", "run-2", "completed"], + ["request-3", "run-3", "completed"], + ]); + }); +}); diff --git a/apps/api/src/routes/taxonomy-readout.ts b/apps/api/src/routes/taxonomy-readout.ts new file mode 100644 index 000000000..6fea6a727 --- /dev/null +++ b/apps/api/src/routes/taxonomy-readout.ts @@ -0,0 +1,345 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash, randomUUID } from "node:crypto"; +import { promisify } from "node:util"; +import { gunzip } from "node:zlib"; +import { z } from "zod"; +import { ExperimentStore, type ExperimentRequestState } from "shared"; +import { apiRoute } from "../openapi/api-route.js"; +import type { RouteContext } from "../route-context.js"; +import { + ProjectIdQuerySchema, + getQueryProjectId, +} from "../utils/project-scope.js"; +import { resolveRunForRequest } from "./requests/resolve-run.js"; + +const gunzipAsync = promisify(gunzip); + +const TaxonomyReadoutReferenceSchema = z + .object({ + status: z.enum(["processing", "done", "failed"]), + experimentExecutionId: z.string(), + taxonomyRevisionId: z.string(), + jsonUrl: z.string().optional(), + markdownUrl: z.string().optional(), + updatedAt: z.coerce.date(), + error: z.string().optional(), + }) + .openapi("TaxonomyReadoutReference"); + +function experimentStore(ctx: RouteContext): ExperimentStore { + return new ExperimentStore({ + experiments: ctx.experimentCollection, + executions: ctx.experimentExecutionsCollection, + chunks: ctx.experimentChunksCollection, + }); +} + +async function requireEnabled(ctx: RouteContext): Promise { + return Boolean( + await ctx.featureFlagCollection.findOne({ + key: "taxonome", + enabled: true, + }), + ); +} + +function renderMarkdown(readout: { + experimentId: string; + experimentExecutionId: string; + taxonomyRef: string; + runs: Array<{ requestId: string; runId: string; state: string }>; +}): string { + const counts = new Map(); + for (const run of readout.runs) { + counts.set(run.state, (counts.get(run.state) ?? 0) + 1); + } + const lines = [ + "# Taxonome readout", + "", + `- Experiment: \`${readout.experimentId}\``, + `- Execution: \`${readout.experimentExecutionId}\``, + `- Taxonomy: \`${readout.taxonomyRef}\``, + `- Runs: ${readout.runs.length}`, + "", + "## States", + "", + ...[...counts.entries()].map(([state, count]) => `- ${state}: ${count}`), + "", + "## Exact runs", + "", + "| Request | Run | State |", + "|---|---|---|", + ...readout.runs.map( + (run) => `| \`${run.requestId}\` | \`${run.runId}\` | ${run.state} |`, + ), + "", + ]; + return lines.join("\n"); +} + +export async function buildTaxonomyReadout( + ctx: RouteContext, + input: { + projectId: string; + experimentId: string; + executionId: string; + requests: ExperimentRequestState[]; + taxonomy: { + taxonomyRevisionId: string; + ref: string; + contentSha256: string; + }; + }, +) { + const runs: Array> = []; + for (const pair of input.requests) { + const resolved = await resolveRunForRequest( + ctx, + pair.requestId, + pair.runId, + ); + if ("error" in resolved) { + runs.push({ + requestId: pair.requestId, + runId: pair.runId, + state: "missing", + }); + continue; + } + const handler = resolved.run.handlerStatus?.["pp-taxonomy"]; + if (handler?.status === "failed") { + runs.push({ + requestId: pair.requestId, + runId: pair.runId, + state: "runtime-failure", + error: handler.error, + }); + continue; + } + const reference = resolved.run.taxonomyEvaluation; + if (!reference) { + runs.push({ + requestId: pair.requestId, + runId: pair.runId, + state: "missing", + }); + continue; + } + const compressed = await ctx.blobStorage.downloadBuffer(reference.url); + const body = await gunzipAsync(compressed); + const hash = createHash("sha256").update(body).digest("hex"); + if (hash !== reference.contentSha256) { + runs.push({ + requestId: pair.requestId, + runId: pair.runId, + state: "runtime-failure", + error: "Taxonomy evaluation integrity check failed", + }); + continue; + } + const result: unknown = JSON.parse(body.toString("utf8")); + runs.push({ + requestId: pair.requestId, + runId: pair.runId, + state: "completed", + result, + }); + } + return { + schemaVersion: "scope/taxonomy-experiment-readout/v1", + projectId: input.projectId, + experimentId: input.experimentId, + experimentExecutionId: input.executionId, + taxonomy: input.taxonomy, + runs, + generatedAt: new Date().toISOString(), + }; +} + +/** Registers typed Taxonome readout status, generation, content, and download routes. */ +export function registerTaxonomyReadoutRoutes(ctx: RouteContext): void { + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: "/api/v1/experiments/:id/taxonomy-readout", + tags: ["Experiments"], + summary: "Get typed Taxonome readout status", + query: ProjectIdQuerySchema, + response: TaxonomyReadoutReferenceSchema, + handler: async (req, res, next) => { + try { + if (!(await requireEnabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const experiment = await experimentStore(ctx).get( + getQueryProjectId(req), + req.params.id, + ); + if (!experiment?.execution.taxonomyReadout) { + res.status(404).json({ error: "Taxonome readout not available" }); + return; + } + res.json(experiment.execution.taxonomyReadout); + } catch (error) { + next(error); + } + }, + }); + + apiRoute(ctx.app, ctx.registry, { + method: "post", + path: "/api/v1/experiments/:id/taxonomy-readout/retry", + tags: ["Experiments"], + summary: "Generate or retry the typed Taxonome readout", + query: ProjectIdQuerySchema, + response: TaxonomyReadoutReferenceSchema, + successStatus: 202, + handler: async (req, res, next) => { + const projectId = getQueryProjectId(req); + const store = experimentStore(ctx); + const claimToken = randomUUID(); + try { + if (!(await requireEnabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const experiment = await store.get(projectId, req.params.id); + const taxonomy = experiment?.execution.taxonomy; + if (!experiment || !taxonomy) { + res.status(404).json({ error: "Taxonomy-enabled Experiment not found" }); + return; + } + const claim = await ctx.experimentCollection.findOneAndUpdate( + { + _id: experiment._id, + projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.version": experiment.execution.version, + }, + { + $set: { + "execution.taxonomyReadout": { + status: "processing", + claimToken, + experimentExecutionId: + experiment.execution.experimentExecutionId, + taxonomyRevisionId: taxonomy.taxonomyRevisionId, + updatedAt: new Date(), + }, + }, + $inc: { "execution.version": 1 }, + }, + { returnDocument: "after" }, + ); + if (!claim) { + res.status(409).json({ error: "Experiment execution changed" }); + return; + } + const requests = await store.readExecutionRequests({ + projectId, + experimentId: experiment.experimentId, + experimentExecutionId: experiment.execution.experimentExecutionId, + }); + if (!requests) throw new Error("Experiment request vector is unavailable"); + const readout = await buildTaxonomyReadout(ctx, { + projectId, + experimentId: experiment.experimentId, + executionId: experiment.execution.experimentExecutionId, + requests, + taxonomy, + }); + const prefix = + `${experiment.experimentId}/executions/` + + `${experiment.execution.experimentExecutionId}/taxonomy-readout/${claimToken}`; + const jsonUrl = await ctx.blobStorage.uploadJson( + `${prefix}/readout.json`, + readout, + ); + const markdownUrl = await ctx.blobStorage.uploadText( + `${prefix}/readout.md`, + renderMarkdown(readout as never), + "text/markdown; charset=utf-8", + ); + const completed = { + status: "done" as const, + experimentExecutionId: experiment.execution.experimentExecutionId, + taxonomyRevisionId: taxonomy.taxonomyRevisionId, + jsonUrl, + markdownUrl, + updatedAt: new Date(), + }; + const publication = await ctx.experimentCollection.updateOne( + { + _id: experiment._id, + projectId, + "execution.experimentExecutionId": + experiment.execution.experimentExecutionId, + "execution.taxonomyReadout.claimToken": claimToken, + }, + { $set: { "execution.taxonomyReadout": completed } }, + ); + if (publication.matchedCount !== 1) { + res.status(409).json({ error: "Experiment execution changed" }); + return; + } + res.status(202).json(completed); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await ctx.experimentCollection.updateOne( + { + _id: req.params.id, + projectId, + "execution.taxonomyReadout.claimToken": claimToken, + }, + { + $set: { + "execution.taxonomyReadout.status": "failed", + "execution.taxonomyReadout.error": message, + "execution.taxonomyReadout.updatedAt": new Date(), + }, + }, + ); + next(error); + } + }, + }); + + for (const format of ["json", "markdown"] as const) { + apiRoute(ctx.app, ctx.registry, { + method: "get", + path: `/api/v1/experiments/:id/taxonomy-readout/${format}`, + tags: ["Experiments"], + summary: `Download Taxonome readout ${format}`, + query: ProjectIdQuerySchema, + response: z.any(), + rawResponse: true, + handler: async (req, res, next) => { + try { + if (!(await requireEnabled(ctx))) { + res.status(404).json({ error: "Taxonome is not enabled" }); + return; + } + const experiment = await experimentStore(ctx).get( + getQueryProjectId(req), + req.params.id, + ); + const reference = experiment?.execution.taxonomyReadout; + const url = format === "json" ? reference?.jsonUrl : reference?.markdownUrl; + if (!url) { + res.status(404).json({ error: "Taxonome readout not available" }); + return; + } + const body = await ctx.blobStorage.downloadBuffer(url); + res + .type(format === "json" ? "application/json" : "text/markdown") + .send(body); + } catch (error) { + next(error); + } + }, + }); + } +} diff --git a/apps/api/src/taxonomy-builtin-seeder.test.ts b/apps/api/src/taxonomy-builtin-seeder.test.ts new file mode 100644 index 000000000..7e4d9fc52 --- /dev/null +++ b/apps/api/src/taxonomy-builtin-seeder.test.ts @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it, vi } from "vitest"; +import { ensureBuiltinTaxonomy } from "./taxonomy-builtin-seeder.js"; + +describe("ensureBuiltinTaxonomy", () => { + it("derives the active root identity, display, version, and ref from YAML", async () => { + const elementRoots = new Map(); + const taxonomyStore = { + getByBusinessId: vi.fn(async () => null), + create: vi.fn(async (input: { id: string; name: string; description: string }) => ({ + _id: "taxonomy-root", + projectId: "project-1", + revisionCounter: 0, + sourceType: "builtin", + createdAt: new Date(), + ...input, + })), + }; + const taxonomyElementStore = { + getByBusinessId: vi.fn(async (_projectId: string, id: string) => + elementRoots.get(id) ?? null + ), + create: vi.fn(async (input: { id: string }) => { + const root = { _id: `root-${input.id}`, id: input.id }; + elementRoots.set(input.id, root); + return root; + }), + }; + const taxonomyElementRevisionStore = { + getLatest: vi.fn(async () => null), + create: vi.fn(async (_rootId: string, content: { id: string }) => ({ + revision: { ref: `${content.id}@r1` }, + deduplicated: false, + })), + }; + const taxonomyRevisionStore = { + create: vi.fn(async ( + _rootId: string, + content: { id: string; version: string }, + ) => ({ + revision: { ref: `${content.id}@r1`, ...content }, + deduplicated: false, + })), + }; + + await ensureBuiltinTaxonomy({ + taxonomyStore, + taxonomyElementStore, + taxonomyElementRevisionStore, + taxonomyRevisionStore, + } as never, "project-1"); + + expect(taxonomyStore.create).toHaveBeenCalledWith( + expect.objectContaining({ + id: "ax-taxonomy", + name: "ax-taxonomy", + description: "ax-taxonomy 1.0.0", + }), + ); + expect(taxonomyRevisionStore.create).toHaveBeenCalledWith( + "taxonomy-root", + expect.objectContaining({ + id: "ax-taxonomy", + version: "1.0.0", + elements: expect.arrayContaining([ + { ref: "beh-discovered@r1" }, + { ref: "beh-timeout@r1" }, + ]), + }), + "system", + ); + }); +}); diff --git a/apps/api/src/taxonomy-builtin-seeder.ts b/apps/api/src/taxonomy-builtin-seeder.ts new file mode 100644 index 000000000..37874a017 --- /dev/null +++ b/apps/api/src/taxonomy-builtin-seeder.ts @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parse as parseYaml } from "yaml"; +import { + PortableTaxonomyElementSchema, + PortableTaxonomySchema, +} from "shared"; +import type { RouteContext } from "./route-context.js"; + +const CATALOG_DIRECTORY = fileURLToPath( + new URL("../../../packages/ax-taxonomy/taxonomy/", import.meta.url), +); + +/** Idempotently seeds the canonical built-in taxonomy into one project. */ +export async function ensureBuiltinTaxonomy( + ctx: RouteContext, + projectId: string, + catalogDirectory = CATALOG_DIRECTORY, +): Promise { + const taxonomyPath = resolve(catalogDirectory, "taxonomy.yaml"); + const taxonomy = PortableTaxonomySchema.parse( + parseYaml(await readFile(taxonomyPath, "utf8")), + ); + const existing = await ctx.taxonomyStore.getByBusinessId( + projectId, + taxonomy.id, + true, + ); + if (existing) return; + + const elementRefs: Array<{ ref: string }> = []; + for (const reference of taxonomy.elements) { + if (!("path" in reference)) { + throw new Error("Canonical built-in taxonomy must use local element paths"); + } + const element = PortableTaxonomyElementSchema.parse( + parseYaml( + await readFile(resolve(dirname(taxonomyPath), reference.path), "utf8"), + ), + ); + const existingElement = await ctx.taxonomyElementStore.getByBusinessId( + projectId, + element.id, + true, + ); + const root = + existingElement ?? + (await ctx.taxonomyElementStore.create({ + projectId, + id: element.id, + name: element.label, + ...(element.description ? { description: element.description } : {}), + sourceType: "builtin", + creator: "system", + })); + const revision = existingElement + ? { + revision: await ctx.taxonomyElementRevisionStore.getLatest(root._id), + deduplicated: true, + } + : await ctx.taxonomyElementRevisionStore.create( + root._id, + element, + "system", + ); + if (!revision.revision) { + throw new Error(`Built-in element '${element.id}' has no revision`); + } + elementRefs.push({ ref: revision.revision.ref }); + } + + const root = await ctx.taxonomyStore.create({ + projectId, + id: taxonomy.id, + name: taxonomy.id, + description: `${taxonomy.id} ${taxonomy.version}`, + sourceType: "builtin", + creator: "system", + }); + await ctx.taxonomyRevisionStore.create( + root._id, + { ...taxonomy, elements: elementRefs }, + "system", + ); +} diff --git a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap index ba858e01b..05316b9be 100644 --- a/apps/cli/src/__snapshots__/cli-commands.test.ts.snap +++ b/apps/cli/src/__snapshots__/cli-commands.test.ts.snap @@ -240,6 +240,7 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` "--resources", "--scenario", "--skills", + "--taxonomy-revision", "--traits", "--url", "--worker", @@ -1211,6 +1212,105 @@ exports[`CLI command registration > full command tree matches snapshot 1`] = ` }, }, }, + "taxonomy": { + "subcommands": { + "copy": { + "options": [ + "--id", + "--name", + "--project", + "--url", + ], + }, + "create": { + "options": [ + "--description", + "--file", + "--name", + "--project", + "--url", + ], + }, + "delete": { + "options": [ + "--url", + ], + }, + "element": { + "subcommands": { + "create": { + "options": [ + "--file", + "--name", + "--project", + "--url", + ], + }, + "list": { + "options": [ + "--project", + "--url", + ], + }, + "revise": { + "options": [ + "--file", + "--url", + ], + }, + "revisions": { + "options": [ + "--url", + ], + }, + }, + }, + "export": { + "options": [ + "--output", + "--url", + ], + }, + "import": { + "options": [ + "--element", + "--name", + "--project", + "--taxonomy", + "--url", + ], + }, + "list": { + "options": [ + "--project", + "--url", + ], + }, + "restore": { + "options": [ + "--url", + ], + }, + "revise": { + "options": [ + "--file", + "--url", + ], + }, + "revisions": { + "options": [ + "--url", + ], + }, + "show": { + "options": [ + "--project", + "--revision", + "--url", + ], + }, + }, + }, "update": { "options": [ "--force", diff --git a/apps/cli/src/commands/experiment.ts b/apps/cli/src/commands/experiment.ts index 44fc623e9..5cbdad806 100644 --- a/apps/cli/src/commands/experiment.ts +++ b/apps/cli/src/commands/experiment.ts @@ -696,6 +696,10 @@ export function registerExperimentCommands(program: Command): void { collectRepeatable, [], ) + .option( + "--taxonomy-revision ", + "Pin one exact taxonomy revision database ID", + ) .option("--empty", "Create an explicit empty experiment") .option( "--request-id ", @@ -768,6 +772,9 @@ export function registerExperimentCommands(program: Command): void { requests: [], requestIds: [], reportTemplateIds: options.reportTemplate as string[], + ...(options.taxonomyRevision + ? { taxonomyRevisionId: options.taxonomyRevision as string } + : {}), } : requestIds.length > 0 ? { @@ -776,6 +783,9 @@ export function registerExperimentCommands(program: Command): void { requests: [], requestIds, reportTemplateIds: options.reportTemplate as string[], + ...(options.taxonomyRevision + ? { taxonomyRevisionId: options.taxonomyRevision as string } + : {}), } : { mode: "compose" as const, @@ -783,6 +793,9 @@ export function registerExperimentCommands(program: Command): void { requests: [built!.request], requestIds: [], reportTemplateIds: options.reportTemplate as string[], + ...(options.taxonomyRevision + ? { taxonomyRevisionId: options.taxonomyRevision as string } + : {}), }; const expandedRunCount = built ? built.profileCount * built.occurrences diff --git a/apps/cli/src/commands/taxonomy.ts b/apps/cli/src/commands/taxonomy.ts new file mode 100644 index 000000000..676da4d34 --- /dev/null +++ b/apps/cli/src/commands/taxonomy.ts @@ -0,0 +1,413 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { Command } from "commander"; +import { parse as parseYaml } from "yaml"; +import { configureHelp } from "../utils/helpFormatter.js"; +import { + getDefaultApiUrl, + normalizeUrl, + withProjectOption, +} from "../utils/shared.js"; +import { requireProjectId } from "../utils/config.js"; +import { errorText, successText } from "../utils/style.js"; + +interface ApiError { + error?: string; +} + +async function responseBody(response: Response): Promise { + const text = await response.text(); + if (!text) return undefined; + try { + return JSON.parse(text) as unknown; + } catch { + return text; + } +} + +async function request( + baseUrl: string, + path: string, + init?: RequestInit, +): Promise { + const response = await fetch(`${normalizeUrl(baseUrl)}/api/v1${path}`, init); + const body = await responseBody(response); + if (!response.ok) { + const error = body as ApiError | undefined; + throw new Error(error?.error ?? `HTTP ${response.status}`); + } + return body; +} + +function projectPath(path: string, projectId: string): string { + const separator = path.includes("?") ? "&" : "?"; + return `${path}${separator}projectId=${encodeURIComponent(projectId)}`; +} + +function print(value: unknown): void { + console.log(JSON.stringify(value, null, 2)); +} + +function contentFromFile(path: string): unknown { + return parseYaml(readFileSync(resolve(path), "utf8")) as unknown; +} + +/** Registers API-backed taxonomy and taxonomy-element commands. */ +export function registerTaxonomyCommands(program: Command): void { + const taxonomy = program + .command("taxonomy") + .description("Manage Taxonome taxonomies and immutable revisions") + .action(() => taxonomy.help()); + configureHelp(taxonomy); + + withProjectOption( + taxonomy + .command("list") + .description("List project taxonomies") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (options) => { + try { + print( + await request( + options.url, + projectPath("/taxonomies", requireProjectId(options.project)), + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withProjectOption( + taxonomy + .command("show") + .description("Show taxonomy detail with exact ordered element revisions") + .argument("", "Taxonomy root database ID") + .option("--revision ", "Exact taxonomy revision ID, ref, or number") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (id, options) => { + try { + const detailPath = options.revision + ? `/taxonomies/${encodeURIComponent(id)}/detail?revisionId=${encodeURIComponent(options.revision)}` + : `/taxonomies/${encodeURIComponent(id)}/detail`; + print( + await request( + options.url, + projectPath(detailPath, requireProjectId(options.project)), + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withProjectOption( + taxonomy + .command("create") + .description("Create a user taxonomy with revision 1") + .requiredOption("--name ", "Display name") + .requiredOption("--file ", "Portable taxonomy YAML") + .option("--description ", "Description") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (options) => { + try { + const body = { + name: options.name, + ...(options.description ? { description: options.description } : {}), + sourceType: "user", + content: contentFromFile(options.file), + }; + print( + await request( + options.url, + projectPath("/taxonomies", requireProjectId(options.project)), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withProjectOption( + taxonomy + .command("import") + .description("Import taxonomy YAML and referenced element YAML") + .requiredOption("--name ", "Display name") + .requiredOption("--taxonomy ", "Taxonomy YAML path") + .option( + "--element ", + "Map a manifest path to an element YAML file (repeatable)", + (value, previous: string[]) => [...previous, value], + [], + ) + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (options) => { + try { + const elements = Object.fromEntries( + (options.element as string[]).map((mapping) => { + const separator = mapping.indexOf("="); + if (separator <= 0) throw new Error(`Invalid --element mapping '${mapping}'`); + return [ + mapping.slice(0, separator), + readFileSync(resolve(mapping.slice(separator + 1)), "utf8"), + ]; + }), + ); + print( + await request( + options.url, + projectPath("/taxonomies/import", requireProjectId(options.project)), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name: options.name, + taxonomyYaml: readFileSync(resolve(options.taxonomy), "utf8"), + elements, + }), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + taxonomy + .command("revisions") + .description("List immutable revisions") + .argument("", "Taxonomy root database ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + print( + await request( + options.url, + `/taxonomies/${encodeURIComponent(id)}/revisions`, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + taxonomy + .command("revise") + .description("Append an immutable user taxonomy revision") + .argument("", "Taxonomy root database ID") + .requiredOption("--file ", "Portable taxonomy YAML") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + print( + await request( + options.url, + `/taxonomies/${encodeURIComponent(id)}/revisions`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ content: contentFromFile(options.file) }), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withProjectOption( + taxonomy + .command("copy") + .description("Copy an exact built-in revision to a user taxonomy") + .argument("", "Built-in taxonomy revision database ID") + .requiredOption("--name ", "New taxonomy name") + .option("--id ", "Portable taxonomy business ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (revisionId, options) => { + try { + print( + await request( + options.url, + projectPath( + `/taxonomy-revisions/${encodeURIComponent(revisionId)}/copy`, + requireProjectId(options.project), + ), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name: options.name, + ...(options.id ? { id: options.id } : {}), + }), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + taxonomy + .command("export") + .description("Export an exact taxonomy revision as YAML") + .argument("", "Taxonomy revision database ID") + .requiredOption("-o, --output ", "Destination YAML file") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (revisionId, options) => { + try { + const response = await fetch( + `${normalizeUrl(options.url)}/api/v1/taxonomy-revisions/${encodeURIComponent(revisionId)}/export`, + ); + if (!response.ok) throw new Error(String(await responseBody(response))); + writeFileSync(resolve(options.output), await response.text(), "utf8"); + console.log(successText(`Exported taxonomy revision to ${options.output}`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + for (const action of ["delete", "restore"] as const) { + taxonomy + .command(action) + .description(`${action === "delete" ? "Soft-delete" : "Restore"} a user taxonomy`) + .argument("", "Taxonomy root database ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + const path = + action === "delete" + ? `/taxonomies/${encodeURIComponent(id)}` + : `/taxonomies/${encodeURIComponent(id)}/restore`; + const result = await request(options.url, path, { + method: action === "delete" ? "DELETE" : "POST", + }); + if (result !== undefined) print(result); + else console.log(successText(`Taxonomy ${action}d.`)); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + } + + const element = taxonomy + .command("element") + .description("Manage taxonomy elements") + .action(() => element.help()); + configureHelp(element); + + withProjectOption( + element + .command("list") + .description("List project taxonomy elements") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (options) => { + try { + print( + await request( + options.url, + projectPath( + "/taxonomy-elements", + requireProjectId(options.project), + ), + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + withProjectOption( + element + .command("create") + .description("Create a user taxonomy element with revision 1") + .requiredOption("--name ", "Display name") + .requiredOption("--file ", "Portable element YAML") + .option("-u, --url ", "API base URL", getDefaultApiUrl()), + ).action(async (options) => { + try { + print( + await request( + options.url, + projectPath( + "/taxonomy-elements", + requireProjectId(options.project), + ), + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name: options.name, + sourceType: "user", + content: contentFromFile(options.file), + }), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + element + .command("revisions") + .description("List immutable taxonomy element revisions") + .argument("", "Element root database ID") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + print( + await request( + options.url, + `/taxonomy-elements/${encodeURIComponent(id)}/revisions`, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); + + element + .command("revise") + .description("Append an immutable user element revision") + .argument("", "Element root database ID") + .requiredOption("--file ", "Portable element YAML") + .option("-u, --url ", "API base URL", getDefaultApiUrl()) + .action(async (id, options) => { + try { + print( + await request( + options.url, + `/taxonomy-elements/${encodeURIComponent(id)}/revisions`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ content: contentFromFile(options.file) }), + }, + ), + ); + } catch (error) { + console.error(errorText("Error:"), error instanceof Error ? error.message : error); + process.exitCode = 1; + } + }); +} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index a9278391a..c235fdfc0 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -26,6 +26,7 @@ import { registerProfileCommands } from "./commands/profile.js"; import { registerProjectCommands } from "./commands/project.js"; import { registerSecretCommands } from "./commands/secret.js"; import { registerExperimentCommands } from "./commands/experiment.js"; +import { registerTaxonomyCommands } from "./commands/taxonomy.js"; import { registerUpdateCommand } from "./commands/update.js"; import { checkForUpdates } from "./utils/update-check.js"; import { errorText } from "./utils/style.js"; @@ -72,6 +73,7 @@ registerSecretCommands(program); registerProjectCommands(program); registerRunCommands(program); registerExperimentCommands(program); +registerTaxonomyCommands(program); registerCriteriaCommands(program); registerPromptFeatureCommands(program); registerReportCommands(program); diff --git a/apps/judge/src/feedback-generator.ts b/apps/judge/src/feedback-generator.ts index ca57c3e8a..65f87d73e 100644 --- a/apps/judge/src/feedback-generator.ts +++ b/apps/judge/src/feedback-generator.ts @@ -3,6 +3,7 @@ import { CopilotClient, SessionEvent } from "@github/copilot-sdk"; import { CriteriaConfig, CriterionResult, DependencyGraph, TokenManagerClient } from "shared"; +import { resolveFeedbackModel } from "./model-config.js"; export interface FeedbackContext { judgeResults: CriterionResult[]; @@ -147,7 +148,7 @@ export class FeedbackGenerator { private tokenClient: TokenManagerClient; constructor(model?: string) { - this.model = model || process.env.FEEDBACK_MODEL || "gpt-5.4-mini"; + this.model = resolveFeedbackModel(model); this.tokenClient = new TokenManagerClient(); } diff --git a/apps/judge/src/judge-evaluation.integration.test.ts b/apps/judge/src/judge-evaluation.integration.test.ts index 8b5f7237f..f8799187f 100644 --- a/apps/judge/src/judge-evaluation.integration.test.ts +++ b/apps/judge/src/judge-evaluation.integration.test.ts @@ -35,7 +35,7 @@ const hasToken = !!process.env.GITHUB_TOKEN; * credential is entitled to. Allow CI to override via JUDGE_MODEL and * otherwise pick a broadly-available model. */ -const JUDGE_MODEL = process.env.JUDGE_MODEL || "gpt-5.4-mini"; +const JUDGE_MODEL = process.env.JUDGE_MODEL || "auto"; describe("judge end-to-end evaluation (integration)", () => { let workspaceDir: string; diff --git a/apps/judge/src/judge-strategies.ts b/apps/judge/src/judge-strategies.ts index ea930a898..1cd35e7e3 100644 --- a/apps/judge/src/judge-strategies.ts +++ b/apps/judge/src/judge-strategies.ts @@ -28,6 +28,7 @@ import { type AgentTrajectory, createTrajectoryTools, } from "./agent-trajectory.js"; +import { resolveJudgeModel } from "./model-config.js"; export interface JudgeStrategyContext { workspacePath: string; @@ -281,7 +282,7 @@ export abstract class JudgeStrategy { protected tokenClient: TokenManagerClient; constructor(model?: string) { - this.model = model || process.env.JUDGE_MODEL || "gpt-5.4-mini"; + this.model = resolveJudgeModel(model); this.timeout = parseInt(process.env.JUDGE_TIMEOUT || String(DEFAULT_JUDGE_TIMEOUT)); this.maxRetries = parseInt(process.env.JUDGE_RETRIES || String(DEFAULT_JUDGE_RETRIES)); this.tokenClient = new TokenManagerClient(); diff --git a/apps/judge/src/model-config.test.ts b/apps/judge/src/model-config.test.ts new file mode 100644 index 000000000..935e27b9e --- /dev/null +++ b/apps/judge/src/model-config.test.ts @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { + DEFAULT_JUDGE_MODEL, + resolveFeedbackModel, + resolveJudgeModel, +} from "./model-config.js"; + +describe("Judge model configuration", () => { + it("uses provider-resolved auto selection by default", () => { + expect(DEFAULT_JUDGE_MODEL).toBe("auto"); + expect(resolveJudgeModel(undefined, {})).toBe("auto"); + expect(resolveFeedbackModel(undefined, {})).toBe("auto"); + }); + + it("preserves explicit constructor and environment precedence", () => { + const env = { + JUDGE_MODEL: "judge-env", + FEEDBACK_MODEL: "feedback-env", + }; + + expect(resolveJudgeModel(undefined, env)).toBe("judge-env"); + expect(resolveFeedbackModel(undefined, env)).toBe("feedback-env"); + expect(resolveJudgeModel("judge-explicit", env)).toBe("judge-explicit"); + expect(resolveFeedbackModel("feedback-explicit", env)).toBe( + "feedback-explicit", + ); + }); +}); diff --git a/apps/judge/src/model-config.ts b/apps/judge/src/model-config.ts new file mode 100644 index 000000000..caac8c0f5 --- /dev/null +++ b/apps/judge/src/model-config.ts @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** Provider-resolved model selection used when no explicit Judge model is set. */ +export const DEFAULT_JUDGE_MODEL = "auto"; + +/** Resolves the Judge model with constructor input taking precedence over env. */ +export function resolveJudgeModel( + model: string | undefined, + env: NodeJS.ProcessEnv = process.env, +): string { + return model || env.JUDGE_MODEL || DEFAULT_JUDGE_MODEL; +} + +/** Resolves the feedback model independently while keeping the same safe default. */ +export function resolveFeedbackModel( + model: string | undefined, + env: NodeJS.ProcessEnv = process.env, +): string { + return model || env.FEEDBACK_MODEL || DEFAULT_JUDGE_MODEL; +} diff --git a/apps/judge/src/observation-trajectory.integration.test.ts b/apps/judge/src/observation-trajectory.integration.test.ts index 066c40da3..2633a8200 100644 --- a/apps/judge/src/observation-trajectory.integration.test.ts +++ b/apps/judge/src/observation-trajectory.integration.test.ts @@ -31,7 +31,7 @@ import { fromAtif, fromAtifList } from "./agent-trajectory.js"; import { evaluateWorkspace } from "./judge-agent.js"; const hasToken = !!process.env.GITHUB_TOKEN; -const JUDGE_MODEL = process.env.JUDGE_MODEL || "gpt-5.4-mini"; +const JUDGE_MODEL = process.env.JUDGE_MODEL || "auto"; /** * Realistic ATIF trajectory (the exact shape pp-taxonomy downloads and forwards diff --git a/apps/portal/src/App.tsx b/apps/portal/src/App.tsx index 195a75400..772da2b4a 100644 --- a/apps/portal/src/App.tsx +++ b/apps/portal/src/App.tsx @@ -67,6 +67,8 @@ import { FeatureRoute } from "@/components/FeatureRoute"; import { ProjectGate } from "@/components/ProjectGate"; import { HomeRoute } from "@/components/HomeRoute"; import { useFavicon } from "@/hooks/useFavicon"; +import { TaxonomyList } from "@/pages/TaxonomyList"; +import { TaxonomyDetail } from "@/pages/TaxonomyDetail"; export function App() { useFavicon(); @@ -146,6 +148,8 @@ export function App() { } /> } /> } /> + } /> + } /> } /> } /> } /> diff --git a/apps/portal/src/components/FeatureRoute.tsx b/apps/portal/src/components/FeatureRoute.tsx index 387d73e86..e2cd2aff2 100644 --- a/apps/portal/src/components/FeatureRoute.tsx +++ b/apps/portal/src/components/FeatureRoute.tsx @@ -7,14 +7,18 @@ import type { ReactNode } from "react"; interface FeatureRouteProps { featureKey: string; + failClosed?: boolean; children: ReactNode; } /** Route guard that redirects to /statistics when a feature flag is disabled */ -export function FeatureRoute({ featureKey, children }: FeatureRouteProps) { - const { isFeatureEnabled } = useFeatureFlags(); +export function FeatureRoute({ featureKey, failClosed = false, children }: FeatureRouteProps) { + const { isFeatureEnabled, isFeatureEnabledFailClosed } = useFeatureFlags(); - if (!isFeatureEnabled(featureKey)) { + const enabled = failClosed + ? isFeatureEnabledFailClosed(featureKey) + : isFeatureEnabled(featureKey); + if (!enabled) { return ; } diff --git a/apps/portal/src/components/Layout.stories.tsx b/apps/portal/src/components/Layout.stories.tsx index a784253ef..764608a58 100644 --- a/apps/portal/src/components/Layout.stories.tsx +++ b/apps/portal/src/components/Layout.stories.tsx @@ -118,6 +118,7 @@ export const Expanded: Story = { ).toBeVisible(); // With a project selected, scoped nav is present. await expect(canvas.getByRole("link", { name: "Runs" })).toBeVisible(); + await expect(canvas.queryByRole("link", { name: "Taxonomies" })).toBeNull(); const experimentsLink = canvas.getByRole("link", { name: "Experiments" }); const runsLink = canvas.getByRole("link", { name: "Runs" }); const criteriaLink = canvas.getByRole("link", { name: "Criteria" }); @@ -132,6 +133,32 @@ export const Expanded: Story = { }, }; +export const TaxonomeEnabled: Story = { + parameters: { + msw: { + handlers: [ + ...mswHandlers, + http.get("/api/v1/feature-flags", () => + HttpResponse.json([ + { + key: "taxonome", + label: "Taxonome and Taxonomies", + enabled: true, + updatedAt: "2026-10-28T00:00:00.000Z", + }, + ]), + ), + ], + }, + }, + play: async ({ canvas }) => { + localStorage.setItem("scope:layout:sidebar-expanded", "true"); + await expect( + await canvas.findByRole("link", { name: "Taxonomies" }), + ).toHaveAttribute("href", "/taxonomies"); + }, +}; + // No project selected: only global entries (Projects + Platform) remain; the // scoped groups and the New Run CTA are hidden until a project is picked. export const NoProjectSelected: Story = { diff --git a/apps/portal/src/components/Layout.tsx b/apps/portal/src/components/Layout.tsx index e2bd8adb1..9ecc80d18 100644 --- a/apps/portal/src/components/Layout.tsx +++ b/apps/portal/src/components/Layout.tsx @@ -14,6 +14,7 @@ import { Plug, PanelLeftClose, PanelLeftOpen, + Network, } from "lucide-react"; import { Button } from "@/components/ui/button"; import { @@ -44,6 +45,7 @@ interface NavItem { icon: React.ComponentType<{ className?: string }>; /** When set, this nav item is only shown if the corresponding feature flag is enabled */ featureKey?: string; + failClosed?: boolean; } interface NavGroup { @@ -99,6 +101,7 @@ const navGroups: NavGroup[] = [ { to: "/profiles", label: "Profiles", icon: navigationIcons.profiles, featureKey: "profiles" }, { to: "/skills", label: "Skills", icon: navigationIcons.skills, featureKey: "skills" }, { to: "/codebases", label: "Codebases", icon: navigationIcons.codebases }, + { to: "/taxonomies", label: "Taxonomies", icon: Network, featureKey: "taxonome", failClosed: true }, ], }, { @@ -564,7 +567,7 @@ function NewWorkflowLauncher({ export function Layout() { const location = useLocation(); - const { isFeatureEnabled } = useFeatureFlags(); + const { isFeatureEnabled, isFeatureEnabledFailClosed } = useFeatureFlags(); const { hasProject } = useProjectContext(); const [mobileOpen, setMobileOpen] = useState(false); const [sidebarExpanded, setSidebarExpanded] = useState(() => { @@ -584,7 +587,12 @@ export function Layout() { }, [sidebarExpanded]); const filterByFeature = (items: NavItem[]) => - items.filter((item) => !item.featureKey || isFeatureEnabled(item.featureKey)); + items.filter((item) => { + if (!item.featureKey) return true; + return item.failClosed + ? isFeatureEnabledFailClosed(item.featureKey) + : isFeatureEnabled(item.featureKey); + }); const visibleGroups = useMemo( () => diff --git a/apps/portal/src/components/StatusBadge.test.tsx b/apps/portal/src/components/StatusBadge.test.tsx index 9a416c058..a2f9dbfe1 100644 --- a/apps/portal/src/components/StatusBadge.test.tsx +++ b/apps/portal/src/components/StatusBadge.test.tsx @@ -25,6 +25,11 @@ describe("StatusBadge", () => { ["processing", "Processing"], ["paused", "Paused"], ["done", "Done"], + ["completed", "Completed"], + ["failed", "Failed"], + ["cancelled", "Cancelled"], + ["deleted", "Deleted"], + ["missing", "Missing"], ]; for (const [status, label] of cases) { const { unmount } = render(); diff --git a/apps/portal/src/components/StatusBadge.tsx b/apps/portal/src/components/StatusBadge.tsx index e99468c00..8c6a1ea7c 100644 --- a/apps/portal/src/components/StatusBadge.tsx +++ b/apps/portal/src/components/StatusBadge.tsx @@ -11,12 +11,25 @@ import type { RunStatus, RunOutcome, RunState } from "@/types"; type BadgeVariant = "default" | "secondary" | "destructive" | "outline" | "success" | "warning" | "info" | "purple"; -const statusConfig: Record = { +export type StatusBadgeStatus = + | RunStatus + | "completed" + | "failed" + | "cancelled" + | "deleted" + | "missing"; + +const statusConfig: Record = { pending: { label: "Pending", variant: "secondary" }, queued: { label: "Queued", variant: "purple" }, processing: { label: "Processing", variant: "info" }, paused: { label: "Paused", variant: "warning" }, done: { label: "Done", variant: "success" }, + completed: { label: "Completed", variant: "success" }, + failed: { label: "Failed", variant: "destructive" }, + cancelled: { label: "Cancelled", variant: "warning" }, + deleted: { label: "Deleted", variant: "destructive" }, + missing: { label: "Missing", variant: "outline" }, }; const outcomeConfig: Record = { @@ -71,7 +84,7 @@ export function StatusBadge({ lastHeartbeatAt, startedAt, }: { - status: RunStatus; + status: StatusBadgeStatus; worker?: RunState["worker"]; lastHeartbeatAt?: RunState["lastHeartbeatAt"]; startedAt?: RunState["startedAt"]; @@ -80,7 +93,9 @@ export function StatusBadge({ // Tick once per second while processing so both the live "Ns ago" tooltip // value and the heartbeat-freshness check below advance over time. const now = useNow(status === "processing"); - const pulsing = shouldPulse(status, lastHeartbeatAt, now, startedAt); + const pulsing = + status === "processing" && + shouldPulse(status, lastHeartbeatAt, now, startedAt); const className = pulsing ? "animate-shimmer" : undefined; const staleClassName = status === "processing" && !pulsing ? "opacity-50" : undefined; const badge = {config.label}; diff --git a/apps/portal/src/components/TaxonomeResultView.stories.tsx b/apps/portal/src/components/TaxonomeResultView.stories.tsx new file mode 100644 index 000000000..8fe2ecd70 --- /dev/null +++ b/apps/portal/src/components/TaxonomeResultView.stories.tsx @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect } from "storybook/test"; +import type { TaxonomyRunResult } from "@/lib/taxonomy-result"; +import { TaxonomeResultView } from "./TaxonomeResultView"; + +const base: TaxonomyRunResult = { + run: { + id: "run-demo", + scenario: { id: "request-demo", label: "Scenario", task: "Build a demo" }, + profile: { id: "profile-demo", label: "Demo profile" }, + }, + results: [ + { + status: "succeeded", + taxonomy: { id: "ax-taxonomy", version: "1.0.0", digest: "a".repeat(64) }, + element: { id: "observed", label: "Observed behavior", category_id: "behavior" }, + implementation: { + type: "prompt", + attempts: [{ attempt: 1, kind: "initial", outcome: "succeeded" }], + }, + input: { sha256: "b".repeat(64) }, + result: { + state: "completed", + observed: true, + occurrences: [{ evidence: [{ role: "action", path: "$.events[0]" }] }], + }, + }, + { + status: "succeeded", + taxonomy: { id: "ax-taxonomy", version: "1.0.0", digest: "a".repeat(64) }, + element: { id: "negative", label: "Not observed" }, + implementation: { + type: "deterministic", + handler: "negative", + attempts: [{ attempt: 1, kind: "initial", outcome: "succeeded" }], + }, + input: { sha256: "b".repeat(64) }, + result: { state: "completed", observed: false }, + }, + { + status: "succeeded", + taxonomy: { id: "ax-taxonomy", version: "1.0.0", digest: "a".repeat(64) }, + element: { id: "unknown", label: "Indeterminate" }, + implementation: { + type: "deterministic", + handler: "unknown", + attempts: [{ attempt: 1, kind: "initial", outcome: "succeeded" }], + }, + input: { sha256: "b".repeat(64) }, + result: { state: "indeterminate", reason: "Evidence was incomplete." }, + }, + ], +}; + +const meta = { + title: "Components/TaxonomeResultView", + component: TaxonomeResultView, +} satisfies Meta; +export default meta; +type Story = StoryObj; + +export const Loading: Story = { args: { loading: true } }; +export const MixedAssessments: Story = { + args: { result: base }, + play: async ({ canvas }) => { + await expect(canvas.getByText("Observed behavior")).toBeVisible(); + await expect(canvas.getAllByText("Not observed")[0]).toBeVisible(); + await expect(canvas.getAllByText("Indeterminate")[0]).toBeVisible(); + }, +}; +export const ElementFailure: Story = { + args: { + result: { + ...base, + results: [ + { + ...base.results[0], + status: "failed", + result: undefined, + error: { + code: "invalid_output", + message: "Model output failed validation.", + validation_errors: [ + { + code: "unknown_role", + output_path: "$.occurrences[0].evidence[0]", + role: "unsupported", + message: "The evidence role is not declared.", + }, + ], + }, + }, + ], + }, + }, + play: async ({ canvas }) => { + await expect(canvas.getByText("Model output failed validation.")).toBeVisible(); + await expect( + canvas.getByText("The evidence role is not declared."), + ).toBeVisible(); + }, +}; +export const ArtifactUnavailable: Story = { + args: { error: "The result artifact is missing or corrupt." }, +}; +export const Historical: Story = { + args: { result: base, readOnlyLabel: "Historical" }, +}; +export const Deleted: Story = { + args: { result: base, readOnlyLabel: "Deleted" }, +}; diff --git a/apps/portal/src/components/TaxonomeResultView.test.tsx b/apps/portal/src/components/TaxonomeResultView.test.tsx new file mode 100644 index 000000000..f71d6cf01 --- /dev/null +++ b/apps/portal/src/components/TaxonomeResultView.test.tsx @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import type { TaxonomyRunResult } from "@/lib/taxonomy-result"; +import { TaxonomeResultView } from "./TaxonomeResultView"; + +const result: TaxonomyRunResult = { + run: { + id: "run-1", + scenario: { id: "request-1", label: "Scenario", task: "Task" }, + profile: { id: "profile-1", label: "Profile" }, + }, + results: [ + { + status: "succeeded", + taxonomy: { id: "ax-taxonomy", version: "1.0.0", digest: "a".repeat(64) }, + element: { id: "beh-test", label: "Test behavior" }, + implementation: { + type: "deterministic", + handler: "test", + attempts: [{ attempt: 1, kind: "initial", outcome: "succeeded" }], + }, + input: { sha256: "b".repeat(64) }, + result: { + state: "completed", + observed: true, + occurrences: [{ evidence: [{ role: "action", path: "$.events[0]" }] }], + }, + }, + ], +}; + +describe("TaxonomeResultView", () => { + afterEach(cleanup); + + it("renders typed element outcomes and identities", () => { + render(); + expect(screen.getByText("ax-taxonomy")).toBeTruthy(); + expect(screen.getByText("Test behavior")).toBeTruthy(); + expect(screen.getByText(/Observed/)).toBeTruthy(); + expect(screen.getByText("$.events[0]")).toBeTruthy(); + }); + + it("renders explicit unavailable state", () => { + render(); + expect(screen.getByText("Taxonome result unavailable")).toBeTruthy(); + expect(screen.getByText("Artifact integrity check failed")).toBeTruthy(); + }); +}); diff --git a/apps/portal/src/components/TaxonomeResultView.tsx b/apps/portal/src/components/TaxonomeResultView.tsx new file mode 100644 index 000000000..e0b3fbd68 --- /dev/null +++ b/apps/portal/src/components/TaxonomeResultView.tsx @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { AlertTriangle, CheckCircle2, CircleSlash2, XCircle } from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import type { + TaxonomyAssessment, + TaxonomyEvaluation, + TaxonomyRunResult, +} from "@/lib/taxonomy-result"; + +function AssessmentView({ assessment }: { assessment: TaxonomyAssessment }) { + if (assessment.state === "indeterminate" || assessment.state === "inapplicable") { + return ( +
+ {assessment.state} +

{assessment.reason}

+
+ ); + } + if (!assessment.observed) { + return ( +
+ + Not observed +
+ ); + } + return ( +
+
+ + Observed · {assessment.occurrences.length} occurrence + {assessment.occurrences.length === 1 ? "" : "s"} +
+ {assessment.occurrences.map((occurrence, index) => ( +
+
Occurrence {index + 1}
+ {occurrence.evidence.map((evidence) => ( +
+ {evidence.role}: + {evidence.path} +
+ ))} +
+ ))} +
+ ); +} + +function EvaluationCard({ evaluation }: { evaluation: TaxonomyEvaluation }) { + return ( + + +
+
+ {evaluation.element.label} +
+ {evaluation.element.id} + {evaluation.element.category_id + ? ` · ${evaluation.element.category_id}` + : ""} +
+
+
+ {evaluation.implementation.type} + + {evaluation.status} + +
+
+
+ + {evaluation.implementation.handler && ( +
+ Handler: {evaluation.implementation.handler} +
+ )} + {evaluation.implementation.provider && ( +
+ Provider: {evaluation.implementation.provider.name} + {evaluation.implementation.provider.version + ? ` ${evaluation.implementation.provider.version}` + : ""} +
+ )} +
+ Attempts:{" "} + {evaluation.implementation.attempts.map((attempt) => ( + + {attempt.attempt} {attempt.kind}: {attempt.outcome} + + ))} +
+ {evaluation.status === "succeeded" && evaluation.result ? ( + + ) : ( +
+
+ + {evaluation.error?.code ?? "evaluation_failed"} +
+

{evaluation.error?.message ?? "The element evaluation failed."}

+ {evaluation.error?.validation_errors?.map((diagnostic, index) => ( +
+
+ {diagnostic.code} · {diagnostic.output_path} +
+ {diagnostic.role &&
Role: {diagnostic.role}
} + {diagnostic.evidence_path && ( +
{diagnostic.evidence_path}
+ )} + {diagnostic.message &&
{diagnostic.message}
} +
+ ))} +
+ )} +
+
+ ); +} + +export function TaxonomeResultView({ + result, + loading = false, + error, + readOnlyLabel, +}: { + result?: TaxonomyRunResult; + loading?: boolean; + error?: string; + readOnlyLabel?: "Historical" | "Deleted"; +}) { + if (loading) { + return ( +
+ + + +
+ ); + } + if (error) { + return ( +
+
+ + Taxonome result unavailable +
+ {error} +
+ ); + } + if (!result) return null; + const taxonomy = result.results[0]?.taxonomy; + return ( +
+
+
+
+

Taxonome result

+

+ Run {result.run.id} · {result.run.scenario.label} · {result.run.profile.label} +

+
+ {readOnlyLabel && {readOnlyLabel}} +
+ {taxonomy && ( +
+
Taxonomy {taxonomy.id}
+
Version {taxonomy.version}
+
+ Digest {taxonomy.digest} +
+
+ )} +
+
+ {result.results.map((evaluation) => ( + + ))} +
+
+ ); +} diff --git a/apps/portal/src/components/TaxonomyStatus.stories.tsx b/apps/portal/src/components/TaxonomyStatus.stories.tsx new file mode 100644 index 000000000..1fbdd6480 --- /dev/null +++ b/apps/portal/src/components/TaxonomyStatus.stories.tsx @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect } from "storybook/test"; +import { Badge } from "@/components/ui/badge"; +import { TaxonomyStatus } from "./TaxonomyStatus"; + +const meta = { + title: "Components/TaxonomyStatus", + component: TaxonomyStatus, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Unavailable: Story = { + play: async ({ canvas }) => { + await expect(canvas.getByLabelText("Taxonomy unavailable")).toBeVisible(); + }, +}; +export const Pending: Story = { + args: { summary: { status: "pending", resultAvailable: false } }, + play: async ({ canvas }) => { + await expect(canvas.getByText("Pending")).toBeVisible(); + }, +}; +export const Queued: Story = { + args: { summary: { status: "queued", resultAvailable: false } }, + play: async ({ canvas }) => { + await expect(canvas.getByText("Queued")).toBeVisible(); + }, +}; +export const Processing: Story = { + args: { summary: { status: "processing", resultAvailable: false } }, + play: async ({ canvas }) => { + await expect(canvas.getByText("Processing")).toHaveClass("animate-shimmer"); + }, +}; +export const Completed: Story = { + args: { summary: { status: "done", resultAvailable: true } }, + play: async ({ canvas }) => { + await expect(canvas.getByText("Completed")).toBeVisible(); + }, +}; +export const Failed: Story = { + args: { + summary: { + status: "failed", + resultAvailable: false, + error: "Taxonome execution failed", + }, + }, + play: async ({ canvas }) => { + await expect(canvas.getByText("Failed")).toBeVisible(); + }, +}; +export const ResultUnavailable: Story = { + args: { summary: { status: "done", resultAvailable: false } }, + play: async ({ canvas }) => { + await expect(canvas.getByText("Result unavailable")).toBeVisible(); + }, +}; +export const Historical: Story = { + args: { summary: { status: "done", resultAvailable: true } }, + render: (args) => ( +
+ + Historical +
+ ), + play: async ({ canvas }) => { + await expect(canvas.getByText("Historical")).toBeVisible(); + await expect(canvas.getByText("Completed")).toBeVisible(); + }, +}; +export const Deleted: Story = { + args: { summary: { status: "done", resultAvailable: true } }, + render: (args) => ( +
+ + Deleted +
+ ), + play: async ({ canvas }) => { + await expect(canvas.getByText("Deleted")).toBeVisible(); + await expect(canvas.getByText("Completed")).toBeVisible(); + }, +}; diff --git a/apps/portal/src/components/TaxonomyStatus.test.tsx b/apps/portal/src/components/TaxonomyStatus.test.tsx new file mode 100644 index 000000000..1235f737f --- /dev/null +++ b/apps/portal/src/components/TaxonomyStatus.test.tsx @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import { TaxonomyStatus } from "./TaxonomyStatus"; + +describe("TaxonomyStatus", () => { + afterEach(cleanup); + + it("renders unavailable without a handler entry", () => { + render(); + expect(screen.getByLabelText("Taxonomy unavailable").textContent).toBe("—"); + }); + + it.each([ + ["pending", false, "Pending"], + ["queued", false, "Queued"], + ["processing", false, "Processing"], + ["failed", false, "Failed"], + ["done", true, "Completed"], + ["done", false, "Result unavailable"], + ] as const)("renders %s / result=%s", (status, resultAvailable, label) => { + render(); + expect(screen.getByText(label)).toBeTruthy(); + }); + + it("animates processing", () => { + render( + , + ); + expect(screen.getByText("Processing").className).toContain("animate-shimmer"); + }); +}); diff --git a/apps/portal/src/components/TaxonomyStatus.tsx b/apps/portal/src/components/TaxonomyStatus.tsx new file mode 100644 index 000000000..2d96092c7 --- /dev/null +++ b/apps/portal/src/components/TaxonomyStatus.tsx @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Badge } from "@/components/ui/badge"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import type { TaxonomyHandlerSummary } from "@/types"; + +export function TaxonomyStatus({ + summary, +}: { + summary?: TaxonomyHandlerSummary; +}) { + if (!summary) { + return ( + + — + + ); + } + + const view = + summary.status === "pending" + ? { label: "Pending", variant: "secondary" as const } + : summary.status === "queued" + ? { label: "Queued", variant: "purple" as const } + : summary.status === "processing" + ? { label: "Processing", variant: "info" as const } + : summary.status === "failed" + ? { label: "Failed", variant: "destructive" as const } + : summary.resultAvailable + ? { label: "Completed", variant: "success" as const } + : { label: "Result unavailable", variant: "warning" as const }; + const badge = ( + + {view.label} + + ); + if (!summary.error && !summary.updatedAt) return badge; + + return ( + + + + {badge} + + + {summary.error &&
{summary.error}
} + {summary.updatedAt && ( +
+ Updated {new Date(summary.updatedAt).toLocaleString()} +
+ )} +
+
+
+ ); +} diff --git a/apps/portal/src/components/list-layout/index.ts b/apps/portal/src/components/list-layout/index.ts index 6cfb44d41..525d70416 100644 --- a/apps/portal/src/components/list-layout/index.ts +++ b/apps/portal/src/components/list-layout/index.ts @@ -10,7 +10,12 @@ export type { FilterSectionProps } from "./FilterSection"; export { CheckboxFilterGroup } from "./CheckboxFilterGroup"; export type { CheckboxFilterGroupProps, CheckboxFilterOption } from "./CheckboxFilterGroup"; export { DataTable } from "./DataTable"; -export type { DataTableProps, DataTableColumn, DataTableSelection } from "./DataTable"; +export type { + DataTableProps, + DataTableColumn, + DataTableSelection, + DataTableGrouping, +} from "./DataTable"; export { EmbeddedTableBlock } from "./EmbeddedTableBlock"; export type { EmbeddedTableBlockProps } from "./EmbeddedTableBlock"; export { Pagination } from "./Pagination"; diff --git a/apps/portal/src/components/runs-table/RunTableCells.tsx b/apps/portal/src/components/runs-table/RunTableCells.tsx new file mode 100644 index 000000000..1ecf1748a --- /dev/null +++ b/apps/portal/src/components/runs-table/RunTableCells.tsx @@ -0,0 +1,415 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { ReactNode } from "react"; +import { Link } from "react-router-dom"; +import { Apple, AppWindow } from "lucide-react"; +import { FaLinux } from "react-icons/fa"; +import { AgentBadge } from "@/components/AgentBadge"; +import { ShortId } from "@/components/ShortId"; +import { StatusBadge, OutcomeBadge, type StatusBadgeStatus } from "@/components/StatusBadge"; +import { TaskPromptBadge } from "@/components/TaskPromptBadge"; +import { TaxonomyStatus } from "@/components/TaxonomyStatus"; +import { Badge } from "@/components/ui/badge"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { formatId, truncate } from "@/lib/utils"; +import type { + OsInfo, + Run, + RunOutcome, + RunState, + TaxonomyHandlerSummary, +} from "@/types"; + +export interface RunTableItem { + requestId: string; + runId: string; + task?: string; + criteria?: string[]; + taskPromptId?: string; + experimentId?: string; + workerType?: string; + model?: string; + reasoningEffort?: string; + agentVersion?: string; + os?: OsInfo; + mcpServers?: string[]; + skillRevisions?: string[]; + extensions?: string[]; + profileId?: string; + priority?: number; + status?: StatusBadgeStatus; + outcome?: RunOutcome; + attemptNumber?: number; + worker?: RunState["worker"]; + lastHeartbeatAt?: RunState["lastHeartbeatAt"]; + startedAt?: RunState["startedAt"]; + taxonomyHandler?: TaxonomyHandlerSummary; +} + +export function runTableItemFromRun(run: Run): RunTableItem { + return { + requestId: run._id, + runId: run.run?._id ?? run._id, + task: run.scenario?.task, + criteria: run.scenario?.criteria, + taskPromptId: run.taskPromptId, + experimentId: run.experimentId, + workerType: run.workerType, + model: run.model, + reasoningEffort: run.reasoningEffort, + agentVersion: run.agentVersion, + os: run.run?.os, + mcpServers: run.mcpServers, + skillRevisions: run.skillRevisions ?? run.skills, + extensions: run.extensions, + profileId: run.profileId, + priority: run.priority, + status: run.run?.status, + outcome: run.run?.outcome, + attemptNumber: run.run?.attemptNumber, + worker: run.run?.worker, + lastHeartbeatAt: run.run?.lastHeartbeatAt, + startedAt: run.run?.startedAt, + taxonomyHandler: run.taxonomyHandler, + }; +} + +export function EmptyRunCell() { + return —; +} + +export function RunIdCell({ item }: { item: RunTableItem }) { + return ; +} + +export function RunTaskCell({ item }: { item: RunTableItem }) { + if (!item.task) return ; + if (!item.taskPromptId) { + return ( + + + + {truncate(item.task, 60)} + + + {item.task} + + ); + } + return ( + + + {truncate(item.task, 60)} + + + ); +} + +/** Compact inline badges with overflow for dense table cells. */ +export function OverflowBadges({ + items, + max = 1, + renderItem, + renderMenuItem, +}: { + items: string[]; + max?: number; + renderItem: (item: string) => ReactNode; + renderMenuItem: (item: string) => ReactNode; +}) { + const visible = items.slice(0, max); + const hidden = items.slice(max); + return ( +
+ {visible.map((item) => renderItem(item))} + {hidden.length > 0 && ( + + + + + + {hidden.map((item) => renderMenuItem(item))} + + + )} +
+ ); +} + +export function RunCriteriaCell({ item }: { item: RunTableItem }) { + const criteria = item.criteria ?? []; + if (criteria.length === 0) return ; + return ( + ( + + + event.stopPropagation()} + > + {criterionId} + + + {criterionId} + + )} + renderMenuItem={(criterionId) => ( + + event.stopPropagation()} + > + {criterionId} + + + )} + /> + ); +} + +export function RunWorkerCell({ item }: { item: RunTableItem }) { + if (!item.workerType) return ; + return ( + + ); +} + +export function RunVersionCell({ item }: { item: RunTableItem }) { + if (!item.agentVersion) return ; + return ( + + + + {item.agentVersion} + + + {item.agentVersion} + + ); +} + +export function OsPlatformIcon({ platform }: { platform: string }) { + const normalized = platform.toLowerCase(); + const Icon = + normalized === "darwin" || normalized === "macos" + ? Apple + : normalized === "win32" || normalized === "windows" + ? AppWindow + : normalized === "linux" + ? FaLinux + : null; + const label = + normalized === "darwin" || normalized === "macos" + ? "macOS" + : normalized === "win32" || normalized === "windows" + ? "Windows" + : normalized === "linux" + ? "Linux" + : platform; + if (!Icon) { + return ( + + {platform} + + ); + } + return ; +} + +export function RunOsCell({ item }: { item: RunTableItem }) { + if (!item.os) return ; + return ( + + + + + + + + {item.os.platform} {item.os.release} ({item.os.arch}) + + + ); +} + +function LinkedValuesCell({ + values, + href, + label, +}: { + values: string[]; + href: (value: string) => string; + label: (value: string) => string; +}) { + if (values.length === 0) return ; + return ( + ( + + + event.stopPropagation()} + > + {truncate(label(value), 20)} + + + {value} + + )} + renderMenuItem={(value) => ( + + event.stopPropagation()} + > + {label(value)} + + + )} + /> + ); +} + +export function RunMcpCell({ item }: { item: RunTableItem }) { + return ( + `/mcp-servers/${value}`} + label={(value) => value} + /> + ); +} + +export function RunSkillsCell({ item }: { item: RunTableItem }) { + return ( + `/skills/${value.split("@")[0]}`} + label={(value) => value.split("@")[0].split("/").pop() ?? value} + /> + ); +} + +export function RunExtensionsCell({ item }: { item: RunTableItem }) { + return ( + `/extensions/${value.split("@")[0]}`} + label={(value) => { + const [qualifiedName, version] = value.split("@"); + const shortName = qualifiedName.split(".").pop() ?? value; + return `${shortName}${version ? `@${version}` : ""}`; + }} + /> + ); +} + +export function RunProfileCell({ + item, + profileNameById, +}: { + item: RunTableItem; + profileNameById?: ReadonlyMap; +}) { + if (!item.profileId) return ; + const label = profileNameById?.get(item.profileId) ?? formatId(item.profileId); + return ( + + + + {label} + + + {label} + + ); +} + +export function RunPriorityCell({ item }: { item: RunTableItem }) { + return item.priority == null ? ( + + ) : ( + {item.priority} + ); +} + +export function RunModelCell({ item }: { item: RunTableItem }) { + if (!item.model) return ; + return ( + + + + {item.model} + + + {item.model} + + ); +} + +export function RunEffortCell({ item }: { item: RunTableItem }) { + return item.reasoningEffort ? ( + {item.reasoningEffort} + ) : ( + + ); +} + +export function RunStatusCell({ item }: { item: RunTableItem }) { + return item.status ? ( + + ) : ( + + ); +} + +export function RunOutcomeCell({ item }: { item: RunTableItem }) { + return item.outcome ? : ; +} + +export function RunTaxonomyCell({ item }: { item: RunTableItem }) { + return ; +} + +export function RunAttemptCell({ item }: { item: RunTableItem }) { + return item.attemptNumber == null ? ( + + ) : ( + #{item.attemptNumber} + ); +} diff --git a/apps/portal/src/components/runs-table/RunsTable.test.tsx b/apps/portal/src/components/runs-table/RunsTable.test.tsx new file mode 100644 index 000000000..e0104480f --- /dev/null +++ b/apps/portal/src/components/runs-table/RunsTable.test.tsx @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + RunOutcomeCell, + RunStatusCell, + type RunTableItem, +} from "./RunTableCells"; +import { RunsTable } from "./RunsTable"; +import type { DataTableColumn } from "@/components/list-layout"; + +const items: RunTableItem[] = [ + { + requestId: "request-processing", + runId: "run-processing", + status: "processing", + outcome: undefined, + }, + { + requestId: "request-deleted", + runId: "run-deleted", + status: "deleted", + outcome: "failed", + }, +]; + +const columns: DataTableColumn[] = [ + { + id: "status", + header: "Status", + cell: (item) => , + }, + { + id: "outcome", + header: "Outcome", + cell: (item) => , + }, +]; + +describe("RunsTable", () => { + afterEach(cleanup); + + it("uses the canonical status and outcome widgets", () => { + render( + item.requestId} + />, + ); + + expect(screen.getAllByText("Processing")[0].className).toContain( + "animate-shimmer", + ); + expect(screen.getAllByText("Deleted").length).toBeGreaterThan(0); + expect(screen.getAllByText("Failed").length).toBeGreaterThan(0); + }); + + it("keeps row clicks isolated from selection controls", () => { + const onRowClick = vi.fn(); + const onToggle = vi.fn(); + render( + item.requestId} + onRowClick={onRowClick} + selection={{ + selectedIds: new Set(), + onToggle, + onToggleAll: vi.fn(), + }} + />, + ); + + fireEvent.click(screen.getAllByRole("checkbox", { name: "Select row" })[0]); + expect(onToggle).toHaveBeenCalledWith( + "request-processing", + items[0], + ); + expect(onRowClick).not.toHaveBeenCalled(); + + fireEvent.click(screen.getAllByText("Processing")[0]); + expect(onRowClick).toHaveBeenCalledWith(items[0]); + }); + + it("renders controlled pagination through the shared shell", () => { + const onPageChange = vi.fn(); + render( + item.requestId} + pagination={{ + page: 1, + pageSize: 1, + total: 2, + onPageChange, + onPageSizeChange: vi.fn(), + }} + />, + ); + + fireEvent.click(screen.getByRole("button", { name: "Next page" })); + expect(onPageChange).toHaveBeenCalledWith(2); + }); +}); diff --git a/apps/portal/src/components/runs-table/RunsTable.tsx b/apps/portal/src/components/runs-table/RunsTable.tsx new file mode 100644 index 000000000..4079f6583 --- /dev/null +++ b/apps/portal/src/components/runs-table/RunsTable.tsx @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import type { Key, ReactNode } from "react"; +import { + DataTable, + Pagination, + type DataTableColumn, + type DataTableGrouping, + type DataTableSelection, +} from "@/components/list-layout"; +import { TooltipProvider } from "@/components/ui/tooltip"; + +export interface RunsTablePagination { + page: number; + pageSize: number; + total?: number; + hasNext?: boolean; + hasPrev?: boolean; + itemLabel?: string; + pageSizeOptions?: number[]; + onPageChange: (page: number) => void; + onPageSizeChange: (pageSize: number) => void; +} + +export interface RunsTableProps { + items: T[]; + columns: readonly DataTableColumn[]; + getRowId: (item: T) => string; + activeId?: Key | null; + onRowClick?: (item: T) => void; + selection?: DataTableSelection; + grouping?: DataTableGrouping; + sort?: string | null; + sortDir?: "asc" | "desc"; + onSortChange?: (column: string) => void; + loading?: boolean; + loadingRows?: number; + emptyState?: ReactNode; + density?: "compact" | "comfortable"; + pagination?: RunsTablePagination; +} + +/** Shared run-list shell. Data fetching and page-specific composition stay outside. */ +export function RunsTable({ + pagination, + ...tableProps +}: RunsTableProps) { + return ( + +
+ + {pagination && ( + + )} +
+
+ ); +} diff --git a/apps/portal/src/components/runs-table/index.ts b/apps/portal/src/components/runs-table/index.ts new file mode 100644 index 000000000..314043f02 --- /dev/null +++ b/apps/portal/src/components/runs-table/index.ts @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export * from "./RunTableCells"; +export * from "./RunsTable"; diff --git a/apps/portal/src/components/ui/stories.play.test.tsx b/apps/portal/src/components/ui/stories.play.test.tsx index 06dbf5816..b7a9d4908 100644 --- a/apps/portal/src/components/ui/stories.play.test.tsx +++ b/apps/portal/src/components/ui/stories.play.test.tsx @@ -26,11 +26,13 @@ import * as textarea from "./textarea.stories"; import * as tooltip from "./tooltip.stories"; import * as sonner from "./sonner.stories"; import * as experimentReports from "../ExperimentReportCard.stories"; +import * as taxonomyStatus from "../TaxonomyStatus.stories"; +import * as taxonomeResultView from "../TaxonomeResultView.stories"; const modules: Record> = { alert, alertDialog, checkbox, dialog, dropdownMenu, label, popover, scrollArea, select, separator, sheet, skeleton, switch: sw, table, tabs, textarea, - tooltip, sonner, experimentReports, + tooltip, sonner, experimentReports, taxonomyStatus, taxonomeResultView, }; afterEach(() => cleanup()); diff --git a/apps/portal/src/contexts/FeatureFlagContext.test.ts b/apps/portal/src/contexts/FeatureFlagContext.test.ts new file mode 100644 index 000000000..0a5d512e5 --- /dev/null +++ b/apps/portal/src/contexts/FeatureFlagContext.test.ts @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { featureEnabledFailClosed } from "./FeatureFlagContext.js"; + +describe("featureEnabledFailClosed", () => { + it("stays disabled while loading, missing, failed-empty, or explicitly off", () => { + expect(featureEnabledFailClosed([], true, "taxonome")).toBe(false); + expect(featureEnabledFailClosed([], false, "taxonome")).toBe(false); + expect( + featureEnabledFailClosed( + [{ key: "taxonome", label: "Taxonome", enabled: false, updatedAt: "" }], + false, + "taxonome", + ), + ).toBe(false); + }); + + it("enables only an explicit on state", () => { + expect( + featureEnabledFailClosed( + [{ key: "taxonome", label: "Taxonome", enabled: true, updatedAt: "" }], + false, + "taxonome", + ), + ).toBe(true); + }); +}); diff --git a/apps/portal/src/contexts/FeatureFlagContext.tsx b/apps/portal/src/contexts/FeatureFlagContext.tsx index 640248c1e..4f4171b6c 100644 --- a/apps/portal/src/contexts/FeatureFlagContext.tsx +++ b/apps/portal/src/contexts/FeatureFlagContext.tsx @@ -15,14 +15,26 @@ interface FeatureFlagContextValue { isLoading: boolean; /** Check if a feature is enabled. Returns true by default (fail-open) if flags haven't loaded. */ isFeatureEnabled: (key: string) => boolean; + /** Fail-closed check for sensitive preview features. */ + isFeatureEnabledFailClosed: (key: string) => boolean; } const FeatureFlagContext = createContext({ flags: [], isLoading: true, isFeatureEnabled: () => true, + isFeatureEnabledFailClosed: () => false, }); +export function featureEnabledFailClosed( + flags: FeatureFlag[], + isLoading: boolean, + key: string, +): boolean { + if (isLoading) return false; + return flags.find((flag) => flag.key === key)?.enabled === true; +} + export function FeatureFlagProvider({ children }: { children: ReactNode }) { const auth = useContext(AuthContext); const enabled = !isAuthEnabled || Boolean(auth?.isReady && auth.isAuthenticated); @@ -41,8 +53,12 @@ export function FeatureFlagProvider({ children }: { children: ReactNode }) { return flag?.enabled ?? true; } + function isFeatureEnabledFailClosed(key: string): boolean { + return featureEnabledFailClosed(flags, isLoading, key); + } + return ( - + {children} ); diff --git a/apps/portal/src/lib/api-experiment-create.test.ts b/apps/portal/src/lib/api-experiment-create.test.ts new file mode 100644 index 000000000..8ec3d5228 --- /dev/null +++ b/apps/portal/src/lib/api-experiment-create.test.ts @@ -0,0 +1,178 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { api } from "./api"; +import { setSelectedProjectIdHolder } from "./project-scope"; + +const apiClientMock = vi.hoisted(() => vi.fn()); + +vi.mock("./api-client", () => ({ apiClient: apiClientMock })); + +const experiment = { + experimentId: "experiment-1", + projectId: "project-1", + metadata: { name: "Experiment" }, + revision: { + experimentRevisionId: "revision-1", + revisionNumber: 1, + memberCount: 1, + }, + execution: { + experimentExecutionId: "execution-1", + attemptNumber: 1, + requests: [], + handlerProgress: {}, + blocking: [], + createdAt: "2026-10-29T00:00:00.000Z", + }, + blocking: [], + createdRequestIds: [], + createdAt: "2026-10-29T00:00:00.000Z", +}; + +function response() { + return new Response(JSON.stringify({ experiment, requestIds: [] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +function postedBody(): Record { + const init = apiClientMock.mock.calls[0]![1] as RequestInit; + return JSON.parse(String(init.body)) as Record; +} + +describe("Experiment create API adapter", () => { + beforeEach(() => { + apiClientMock.mockReset(); + apiClientMock.mockResolvedValue(response()); + setSelectedProjectIdHolder("project-1"); + }); + + afterEach(() => { + setSelectedProjectIdHolder(undefined); + }); + + it("maps worker to workerType and expands count without a legacy query", async () => { + await api.createExperiment({ + metadata: { name: "Manual" }, + taxonomyRevisionId: "taxonomy-revision", + request: { + scenario: { task: "Build it", criteria: [] }, + worker: "coder-acp-copilot", + model: "gpt-6.1-sol", + count: 2, + }, + }); + + expect(apiClientMock).toHaveBeenCalledWith( + "/api/v1/experiments?projectId=project-1", + expect.objectContaining({ method: "POST" }), + ); + expect(postedBody()).toEqual({ + mode: "compose", + metadata: { name: "Manual" }, + requests: [ + { + scenario: { task: "Build it", criteria: [] }, + workerType: "coder-acp-copilot", + model: "gpt-6.1-sol", + }, + { + scenario: { task: "Build it", criteria: [] }, + workerType: "coder-acp-copilot", + model: "gpt-6.1-sol", + }, + ], + reportTemplateIds: [], + taxonomyRevisionId: "taxonomy-revision", + }); + }); + + it("expands profile variations with each resolved worker/model target", async () => { + await api.createExperiment({ + metadata: { name: "Profiles" }, + request: { + scenario: { task: "Compare", criteria: [] }, + profileId: "base@2", + profileVariations: ["variation@3"], + count: 2, + experimentProfileTargets: [ + { + profileId: "base@2", + worker: "coder-acp-copilot", + model: "gpt-6.1-sol", + agentVersion: "copilot-dev", + }, + { + profileId: "variation@3", + worker: "coder-acp-claude-code", + model: "claude-sonnet-5.5", + }, + ], + }, + }); + + expect(postedBody().requests).toEqual([ + expect.objectContaining({ + profileId: "base@2", + workerType: "coder-acp-copilot", + model: "gpt-6.1-sol", + agentVersion: "copilot-dev", + }), + expect.objectContaining({ + profileId: "base@2", + workerType: "coder-acp-copilot", + model: "gpt-6.1-sol", + agentVersion: "copilot-dev", + }), + expect.objectContaining({ + profileId: "variation@3", + workerType: "coder-acp-claude-code", + model: "claude-sonnet-5.5", + }), + expect.objectContaining({ + profileId: "variation@3", + workerType: "coder-acp-claude-code", + model: "claude-sonnet-5.5", + }), + ]); + expect(JSON.stringify(postedBody())).not.toContain("profileVariations"); + expect(JSON.stringify(postedBody())).not.toContain('"worker":'); + }); + + it("sends an explicit empty Experiment without requests", async () => { + await api.createExperiment({ + metadata: { name: "Reserved" }, + reportTemplateIds: ["template-1"], + empty: true, + }); + + expect(postedBody()).toEqual({ + mode: "empty", + metadata: { name: "Reserved" }, + requests: [], + reportTemplateIds: ["template-1"], + }); + }); + + it("preserves legacy run submission query composition separately", async () => { + apiClientMock.mockResolvedValue(new Response(JSON.stringify({ id: "run-1" }))); + await api.submitRun({ + scenario: { task: "Build it", criteria: [] }, + worker: "coder-acp-copilot", + model: "gpt-6.1-sol", + }); + + expect(apiClientMock).toHaveBeenCalledWith( + "/api/v1/requests?worker=coder-acp-copilot&projectId=project-1", + expect.objectContaining({ + body: JSON.stringify({ + scenario: { task: "Build it", criteria: [] }, + model: "gpt-6.1-sol", + }), + }), + ); + }); +}); diff --git a/apps/portal/src/lib/api.ts b/apps/portal/src/lib/api.ts index 522df076c..2b0d4e1a2 100644 --- a/apps/portal/src/lib/api.ts +++ b/apps/portal/src/lib/api.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import type { Run, RunState, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTemplateVersion, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, SkillDocument, SkillSearchResult, SkillDiscoveryResult, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, CodebaseSourceType, ResourceDocument, ResourceRevisionDocument, ResourceBindingSpec, CreateResourceBody, CreateResourceRevisionBody, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest, ProfileWithVersion, ProfileVersionDocument, ProfileDocument, RunGroup, RunFacetsResponse, CursorPaginatedResponse, IterationOp, GateConfig, GateId, PromptType, RunSortField, RunSortDir, CriterionKind, TaxonomyElementId, CriterionSubject } from "@/types"; +import type { Run, RunState, CriteriaDocument, CriteriaGraphData, GeneratePromptResponse, AnalysisResponse, PromptFeatureDocument, Report, BulkReportStatus, BulkReportSummary, ReportTemplate, ReportTemplateVersion, ReportTrigger, ReportTemplateSystemPrompt, KeyDocument, KeyValidationResult, CreateKeyRequest, UpdateKeyRequest, CodingAgent, AgentVersion, McpServerDocument, CreateMcpServerRequest, UpdateMcpServerRequest, BulkResubmitOverrides, Insight, InsightWithReference, TaskPrompt, TaskPromptFeatureExtractionResult, Model, FeatureFlag, TaxonomyDocument, TaxonomyListItem, TaxonomyDetailDocument, TaxonomyRevisionDocument, TaxonomyElementDocument, TaxonomyElementRevisionDocument, SkillDocument, SkillSearchResult, SkillDiscoveryResult, SkillRevisionDocument, CodebaseDocument, CodebaseRevisionDocument, CodebaseSourceType, ResourceDocument, ResourceRevisionDocument, ResourceBindingSpec, CreateResourceBody, CreateResourceRevisionBody, ExtensionDocument, ExtensionSearchResult, ExtensionVersionInfo, MdpResponse, AccountDocument, CreateAccountRequest, UpdateAccountRequest, ProfileWithVersion, ProfileVersionDocument, ProfileDocument, RunGroup, RunFacetsResponse, CursorPaginatedResponse, IterationOp, GateConfig, GateId, PromptType, RunSortField, RunSortDir, CriterionKind, TaxonomyElementId, CriterionSubject } from "@/types"; import type { Project, CreateProjectRequest, UpdateProjectRequest } from "@/types"; import { qs } from "./url"; @@ -26,6 +26,10 @@ import { type ExperimentRevisionIdentityView, type ExperimentView, } from "./experiments"; +import { + parseTaxonomyRunResult, + type TaxonomyRunResult, +} from "./taxonomy-result"; const BASE = "/api/v1"; @@ -203,6 +207,61 @@ export interface RunCreateInput { codebase?: string; codebaseRevisionId?: string; priority?: number; + /** Portal-only canonical Experiment expansion targets; never sent directly. */ + experimentProfileTargets?: Array<{ + profileId?: string; + worker: string; + model?: string; + reasoningEffort?: string; + agentVersion?: string; + }>; +} + +/** Expands legacy run composition fields into canonical Experiment requests. */ +export function canonicalExperimentRequests( + input: RunCreateInput, +): Array> { + const { + worker, + model, + reasoningEffort, + agentVersion, + profileId, + profileVariations = [], + count = 1, + experimentProfileTargets, + ...shared + } = input; + const fallbackProfiles = profileVariations.length > 0 + ? [profileId, ...profileVariations].filter( + (value): value is string => Boolean(value), + ) + : [profileId]; + const targets = experimentProfileTargets?.length + ? experimentProfileTargets + : fallbackProfiles.map(targetProfileId => ({ + ...(targetProfileId ? { profileId: targetProfileId } : {}), + worker: worker ?? "", + ...(model ? { model } : {}), + ...(reasoningEffort ? { reasoningEffort } : {}), + ...(agentVersion ? { agentVersion } : {}), + })); + + return targets.flatMap(target => { + if (!target.worker.trim()) { + throw new Error("Experiment request requires a worker"); + } + return Array.from({ length: count }, () => ({ + ...shared, + workerType: target.worker, + ...(target.model ? { model: target.model } : {}), + ...(target.reasoningEffort + ? { reasoningEffort: target.reasoningEffort } + : {}), + ...(target.agentVersion ? { agentVersion: target.agentVersion } : {}), + ...(target.profileId ? { profileId: target.profileId } : {}), + })); + }); } export const api = { @@ -278,28 +337,26 @@ export const api = { }, createExperiment: async ( - body: { metadata: ExperimentMetadataView; reportTemplateIds?: string[] } & ( + body: { metadata: ExperimentMetadataView; reportTemplateIds?: string[]; taxonomyRevisionId?: string } & ( | { request: RunCreateInput; empty?: never } | { empty: true; request?: never } ), ): Promise => { const composedRequest = "request" in body ? body.request : undefined; - const worker = composedRequest?.worker; - const requestBody = composedRequest - ? { ...composedRequest, worker: undefined } - : undefined; - const path = worker - ? `/experiments?worker=${encodeURIComponent(worker)}` - : "/experiments"; const value = await request( - path, + "/experiments", { method: "POST", body: JSON.stringify({ mode: "empty" in body ? "empty" : "compose", metadata: body.metadata, - requests: requestBody ? [requestBody] : [], + requests: composedRequest + ? canonicalExperimentRequests(composedRequest) + : [], reportTemplateIds: body.reportTemplateIds ?? [], + ...(body.taxonomyRevisionId + ? { taxonomyRevisionId: body.taxonomyRevisionId } + : {}), }), }, { scoped: true }, @@ -314,6 +371,7 @@ export const api = { metadata: ExperimentMetadataView; requestIds: string[]; reportTemplateIds?: string[]; + taxonomyRevisionId?: string; }): Promise => { const value = await request( "/experiments", @@ -325,6 +383,9 @@ export const api = { requests: [], requestIds: body.requestIds, reportTemplateIds: body.reportTemplateIds ?? [], + ...(body.taxonomyRevisionId + ? { taxonomyRevisionId: body.taxonomyRevisionId } + : {}), }), }, { scoped: true }, @@ -1396,6 +1457,122 @@ export const api = { }); }, + // ─── Taxonomies ──────────────────────────────────────────────────────────── + + listTaxonomies: (): Promise => { + return request("/taxonomies", undefined, { scoped: true }); + }, + + getTaxonomyDetail: ( + id: string, + revisionId?: string, + ): Promise => { + const query = revisionId + ? `?revisionId=${encodeURIComponent(revisionId)}` + : ""; + return request( + `/taxonomies/${encodeURIComponent(id)}/detail${query}`, + undefined, + { scoped: true }, + ); + }, + + getTaxonomy: (id: string): Promise => { + return request(`/taxonomies/${encodeURIComponent(id)}`); + }, + + listTaxonomyRevisions: (id: string): Promise => { + return request(`/taxonomies/${encodeURIComponent(id)}/revisions`); + }, + + createTaxonomyRevision: ( + id: string, + content: unknown, + ): Promise => { + return request(`/taxonomies/${encodeURIComponent(id)}/revisions`, { + method: "POST", + body: JSON.stringify({ content }), + }); + }, + + createTaxonomy: (input: { + name: string; + description?: string; + content: unknown; + }): Promise => { + return request("/taxonomies", { + method: "POST", + body: JSON.stringify({ ...input, sourceType: "user" }), + }, { scoped: true }); + }, + + copyTaxonomyRevision: ( + revisionId: string, + input: { name: string; id?: string }, + ): Promise => { + return request( + `/taxonomy-revisions/${encodeURIComponent(revisionId)}/copy`, + { method: "POST", body: JSON.stringify(input) }, + { scoped: true }, + ); + }, + + deleteTaxonomy: (id: string): Promise => { + return request(`/taxonomies/${encodeURIComponent(id)}`, { + method: "DELETE", + }); + }, + + listTaxonomyElements: (): Promise => { + return request("/taxonomy-elements", undefined, { scoped: true }); + }, + + listTaxonomyElementRevisions: ( + id: string, + ): Promise => { + return request(`/taxonomy-elements/${encodeURIComponent(id)}/revisions`); + }, + + createTaxonomyElementRevision: ( + id: string, + content: unknown, + ): Promise => { + return request(`/taxonomy-elements/${encodeURIComponent(id)}/revisions`, { + method: "POST", + body: JSON.stringify({ content }), + }); + }, + + getTaxonomyReadout: (experimentId: string): Promise<{ + status: "processing" | "done" | "failed"; + jsonUrl?: string; + markdownUrl?: string; + error?: string; + }> => { + return request(`/experiments/${encodeURIComponent(experimentId)}/taxonomy-readout`, undefined, { scoped: true }); + }, + + retryTaxonomyReadout: (experimentId: string): Promise<{ + status: "processing" | "done" | "failed"; + jsonUrl?: string; + markdownUrl?: string; + error?: string; + }> => { + return request(`/experiments/${encodeURIComponent(experimentId)}/taxonomy-readout/retry`, { + method: "POST", + }, { scoped: true }); + }, + + getTaxonomyEvaluationJson: async ( + requestId: string, + runId: string, + ): Promise => { + const value = await request( + `/requests/${encodeURIComponent(requestId)}/runs/${encodeURIComponent(runId)}/taxonomy-evaluation/json`, + ); + return parseTaxonomyRunResult(value); + }, + // ─── Skills ─────────────────────────────────────────────────────────────── /** List all imported skills */ diff --git a/apps/portal/src/lib/experiments.test.ts b/apps/portal/src/lib/experiments.test.ts index 10857dbf3..348e33891 100644 --- a/apps/portal/src/lib/experiments.test.ts +++ b/apps/portal/src/lib/experiments.test.ts @@ -15,6 +15,15 @@ const resource = { readiness: "done", progress: {}, blockers: [], + requests: [{ + requestId: "request-1", + runId: "run-1", + status: "done", + taxonomyHandler: { + status: "pending", + resultAvailable: false, + }, + }], createdAt: "2026-10-16T12:00:00Z", }, createdAt: "2026-10-16T12:00:00Z", @@ -35,6 +44,10 @@ describe("Experiment operation responses", () => { }); if ("phase" in active) throw new Error("Expected an Experiment resource"); expect(active.lifecycle).toBeUndefined(); + expect(active.execution.requests[0]?.taxonomyHandler).toEqual({ + status: "pending", + resultAvailable: false, + }); expect(parseExperimentOperation(progress)).toMatchObject({ operationId: "operation-1", phase: "restoring", }); diff --git a/apps/portal/src/lib/experiments.ts b/apps/portal/src/lib/experiments.ts index 39c5e47e8..b099ae414 100644 --- a/apps/portal/src/lib/experiments.ts +++ b/apps/portal/src/lib/experiments.ts @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import type { TaxonomyHandlerSummary } from "@/types"; + export const LEGACY_REQUEST_COMPOSITION_DEPRECATION = "legacy-request-composition" as const; @@ -49,11 +51,25 @@ export interface ExperimentRequestSummaryView { requestId: string; runId: string; task?: string; + criteria?: string[]; + taskPromptId?: string; status?: string; outcome?: string; workerType?: string; model?: string; + reasoningEffort?: string; + agentVersion?: string; + os?: { platform: string; release: string; arch: string }; + mcpServers?: string[]; + skillRevisions?: string[]; + extensions?: string[]; profileId?: string; + priority?: number; + attemptNumber?: number; + worker?: { instanceId: string; podName?: string }; + lastHeartbeatAt?: string; + startedAt?: string; + taxonomyHandler?: TaxonomyHandlerSummary; } export interface ExperimentExecutionView { @@ -271,15 +287,93 @@ function parseBlocker(value: unknown): ExperimentBlockerView { function parseRequestSummary(value: unknown): ExperimentRequestSummaryView { const item = record(value, "experiment request"); + const osValue = item.os; + const os = osValue === undefined + ? undefined + : (() => { + const value = record(osValue, "experiment request os"); + return { + platform: requiredString(value, "platform", "experiment request os"), + release: requiredString(value, "release", "experiment request os"), + arch: requiredString(value, "arch", "experiment request os"), + }; + })(); + const workerValue = item.worker; + const worker = workerValue === undefined + ? undefined + : (() => { + const value = record(workerValue, "experiment request worker"); + return { + instanceId: requiredString( + value, + "instanceId", + "experiment request worker", + ), + podName: optionalString(value, "podName"), + }; + })(); + const taxonomyHandlerValue = item.taxonomyHandler; + const taxonomyHandler = taxonomyHandlerValue === undefined + ? undefined + : ((): TaxonomyHandlerSummary => { + const value = record( + taxonomyHandlerValue, + "experiment request taxonomy handler", + ); + const status = requiredString( + value, + "status", + "experiment request taxonomy handler", + ); + if ( + status !== "pending" && + status !== "queued" && + status !== "processing" && + status !== "done" && + status !== "failed" + ) { + throw new Error("Invalid experiment request taxonomy handler status"); + } + return { + status, + updatedAt: optionalString(value, "updatedAt"), + error: optionalString(value, "error"), + resultAvailable: value.resultAvailable === true, + }; + })(); + const strings = (key: string): string[] | undefined => { + const candidate = item[key]; + if (candidate === undefined) return undefined; + if (!Array.isArray(candidate) || candidate.some((value) => typeof value !== "string")) { + throw new Error(`Invalid experiment request.${key} response from Scope`); + } + return candidate; + }; return { requestId: requiredString(item, "requestId", "experiment request"), runId: requiredString(item, "runId", "experiment request"), task: optionalString(item, "task"), + criteria: strings("criteria"), + taskPromptId: optionalString(item, "taskPromptId"), status: optionalString(item, "status"), outcome: optionalString(item, "outcome"), workerType: optionalString(item, "workerType"), model: optionalString(item, "model"), + reasoningEffort: optionalString(item, "reasoningEffort"), + agentVersion: optionalString(item, "agentVersion"), + os, + mcpServers: strings("mcpServers"), + skillRevisions: strings("skillRevisions"), + extensions: strings("extensions"), profileId: optionalString(item, "profileId"), + priority: + typeof item.priority === "number" ? item.priority : undefined, + attemptNumber: + typeof item.attemptNumber === "number" ? item.attemptNumber : undefined, + worker, + lastHeartbeatAt: optionalString(item, "lastHeartbeatAt"), + startedAt: optionalString(item, "startedAt"), + taxonomyHandler, }; } diff --git a/apps/portal/src/lib/run-navigation.test.ts b/apps/portal/src/lib/run-navigation.test.ts new file mode 100644 index 000000000..a38896cbc --- /dev/null +++ b/apps/portal/src/lib/run-navigation.test.ts @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { + isSafeInternalReturnTarget, + resolveRunReturnState, + runNavigationState, +} from "./run-navigation"; + +describe("run navigation state", () => { + it("preserves an internal return target and label", () => { + expect( + resolveRunReturnState( + runNavigationState( + "/experiments/experiment-1?tab=runs&executionId=execution-2", + "Back to experiment", + ), + ), + ).toEqual({ + returnTo: + "/experiments/experiment-1?tab=runs&executionId=execution-2", + returnLabel: "Back to experiment", + }); + }); + + it("preserves Runs query state", () => { + expect( + resolveRunReturnState( + runNavigationState("/runs?groupBy=experimentId&page=2", "Back to runs"), + ), + ).toEqual({ + returnTo: "/runs?groupBy=experimentId&page=2", + returnLabel: "Back to runs", + }); + }); + + it.each([ + undefined, + null, + {}, + { returnTo: "https://example.com", returnLabel: "External" }, + { returnTo: "//example.com", returnLabel: "External" }, + { returnTo: "\\\\example.com", returnLabel: "External" }, + ])("falls back safely for invalid state %#", (state) => { + expect(resolveRunReturnState(state)).toEqual({ + returnTo: "/runs", + returnLabel: "Back to runs", + }); + }); + + it("accepts only same-origin path targets", () => { + expect(isSafeInternalReturnTarget("/runs?status=done")).toBe(true); + expect(isSafeInternalReturnTarget("https://example.com/runs")).toBe(false); + }); +}); diff --git a/apps/portal/src/lib/run-navigation.ts b/apps/portal/src/lib/run-navigation.ts new file mode 100644 index 000000000..3b4d45acb --- /dev/null +++ b/apps/portal/src/lib/run-navigation.ts @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export interface RunNavigationState { + returnTo: string; + returnLabel: string; +} + +export function isSafeInternalReturnTarget(value: unknown): value is string { + return ( + typeof value === "string" && + value.startsWith("/") && + !value.startsWith("//") && + !value.includes("\\") && + !value.includes("://") + ); +} + +export function runNavigationState( + returnTo: string, + returnLabel: string, +): RunNavigationState { + return { returnTo, returnLabel }; +} + +export function resolveRunReturnState( + state: unknown, +): RunNavigationState { + if (!state || typeof state !== "object") { + return { returnTo: "/runs", returnLabel: "Back to runs" }; + } + const candidate = state as Partial; + if (!isSafeInternalReturnTarget(candidate.returnTo)) { + return { returnTo: "/runs", returnLabel: "Back to runs" }; + } + return { + returnTo: candidate.returnTo, + returnLabel: + typeof candidate.returnLabel === "string" && + candidate.returnLabel.trim().length > 0 + ? candidate.returnLabel + : "Back", + }; +} diff --git a/apps/portal/src/lib/taxonomy-result.test.ts b/apps/portal/src/lib/taxonomy-result.test.ts new file mode 100644 index 000000000..1b43b1677 --- /dev/null +++ b/apps/portal/src/lib/taxonomy-result.test.ts @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { parseTaxonomyRunResult } from "./taxonomy-result"; + +const result = { + run: { + id: "run-1", + scenario: { id: "scenario-1", label: "Scenario" }, + profile: { id: "profile-1", label: "Profile" }, + }, + results: [ + { + schema_version: "scope/taxonomy-evaluation/v1", + status: "succeeded", + taxonomy: { + id: "ax-taxonomy", + version: "1.0.0", + digest: "a".repeat(64), + }, + element: { id: "element-1", label: "Element" }, + implementation: { + type: "deterministic", + handler: "element-1", + attempts: [{ attempt: 1, kind: "initial", outcome: "succeeded" }], + }, + input: { + normalization: "scope/taxonome-normalized-atif/v1", + sha256: "b".repeat(64), + }, + result: { state: "completed", observed: false }, + }, + ], +}; + +describe("parseTaxonomyRunResult", () => { + it("parses the typed result envelope", () => { + expect(parseTaxonomyRunResult(result)).toMatchObject({ + run: { id: "run-1" }, + results: [{ status: "succeeded", element: { id: "element-1" } }], + }); + }); + + it.each([ + null, + { ...result, results: null }, + { + ...result, + results: [{ ...result.results[0], status: "unexpected" }], + }, + { + ...result, + results: [{ ...result.results[0], element: {} }], + }, + ])("rejects malformed result data %#", (value) => { + expect(() => parseTaxonomyRunResult(value)).toThrow(); + }); +}); diff --git a/apps/portal/src/lib/taxonomy-result.ts b/apps/portal/src/lib/taxonomy-result.ts new file mode 100644 index 000000000..db5b9ff94 --- /dev/null +++ b/apps/portal/src/lib/taxonomy-result.ts @@ -0,0 +1,263 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export interface TaxonomyEvidenceReference { + role: string; + path: string; +} + +export type TaxonomyAssessment = + | { + state: "completed"; + observed: true; + occurrences: Array<{ evidence: TaxonomyEvidenceReference[] }>; + } + | { state: "completed"; observed: false } + | { state: "indeterminate"; reason: string } + | { state: "inapplicable"; reason: string }; + +export interface TaxonomyEvaluation { + status: "succeeded" | "failed"; + taxonomy: { id: string; version: string; digest: string }; + element: { id: string; label: string; category_id?: string }; + implementation: { + type: "prompt" | "deterministic"; + handler?: string; + provider?: { name: string; version?: string; origin: string }; + attempts: Array<{ attempt: 1 | 2; kind: string; outcome: string }>; + }; + input: Record; + result?: TaxonomyAssessment; + error?: { + code: string; + message: string; + validation_errors?: Array<{ + code: string; + output_path: string; + role?: string; + evidence_path?: string; + message?: string; + }>; + }; +} + +export interface TaxonomyRunResult { + run: { + id: string; + scenario: { id: string; label: string; task?: string }; + profile: { id: string; label: string }; + }; + results: TaxonomyEvaluation[]; + metadata?: unknown; +} + +function object(value: unknown, name: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`Invalid ${name}`); + } + return value as Record; +} + +function string(value: Record, key: string, name: string) { + if (typeof value[key] !== "string" || value[key].length === 0) { + throw new Error(`Invalid ${name}.${key}`); + } + return value[key] as string; +} + +function optionalString( + value: Record, + key: string, +): string | undefined { + const candidate = value[key]; + if (candidate === undefined) return undefined; + if (typeof candidate !== "string") throw new Error(`Invalid ${key}`); + return candidate; +} + +function parseAssessment(value: unknown): TaxonomyAssessment { + const assessment = object(value, "Taxonome assessment"); + const state = string(assessment, "state", "Taxonome assessment"); + if (state === "indeterminate" || state === "inapplicable") { + return { + state, + reason: string(assessment, "reason", "Taxonome assessment"), + }; + } + if (state !== "completed" || typeof assessment.observed !== "boolean") { + throw new Error("Invalid Taxonome assessment state"); + } + if (!assessment.observed) return { state, observed: false }; + if (!Array.isArray(assessment.occurrences)) { + throw new Error("Invalid Taxonome assessment occurrences"); + } + return { + state, + observed: true, + occurrences: assessment.occurrences.map((candidate) => { + const occurrence = object(candidate, "Taxonome occurrence"); + if (!Array.isArray(occurrence.evidence)) { + throw new Error("Invalid Taxonome occurrence evidence"); + } + return { + evidence: occurrence.evidence.map((candidate) => { + const evidence = object(candidate, "Taxonome evidence"); + return { + role: string(evidence, "role", "Taxonome evidence"), + path: string(evidence, "path", "Taxonome evidence"), + }; + }), + }; + }), + }; +} + +/** Validates the browser boundary while retaining the typed result document. */ +export function parseTaxonomyRunResult(value: unknown): TaxonomyRunResult { + const root = object(value, "Taxonome result"); + const run = object(root.run, "Taxonome result.run"); + const scenario = object(run.scenario, "Taxonome result.run.scenario"); + const profile = object(run.profile, "Taxonome result.run.profile"); + if (!Array.isArray(root.results)) { + throw new Error("Invalid Taxonome result.results"); + } + const results = root.results.map((candidate, index) => { + const evaluation = object(candidate, `Taxonome result.results[${index}]`); + const statusValue = string(evaluation, "status", "Taxonome evaluation"); + if (statusValue !== "succeeded" && statusValue !== "failed") { + throw new Error("Invalid Taxonome evaluation.status"); + } + const status: TaxonomyEvaluation["status"] = statusValue; + const taxonomy = object(evaluation.taxonomy, "Taxonome evaluation.taxonomy"); + const element = object(evaluation.element, "Taxonome evaluation.element"); + const implementation = object( + evaluation.implementation, + "Taxonome evaluation.implementation", + ); + if (!Array.isArray(implementation.attempts)) { + throw new Error("Invalid Taxonome evaluation attempts"); + } + const implementationTypeValue = string( + implementation, + "type", + "Taxonome implementation", + ); + if ( + implementationTypeValue !== "prompt" && + implementationTypeValue !== "deterministic" + ) { + throw new Error("Invalid Taxonome implementation.type"); + } + const implementationType: TaxonomyEvaluation["implementation"]["type"] = + implementationTypeValue; + const attempts = implementation.attempts.map((candidate) => { + const attempt = object(candidate, "Taxonome attempt"); + const attemptNumber = attempt.attempt; + if (attemptNumber !== 1 && attemptNumber !== 2) { + throw new Error("Invalid Taxonome attempt.attempt"); + } + return { + attempt: attemptNumber, + kind: string(attempt, "kind", "Taxonome attempt"), + outcome: string(attempt, "outcome", "Taxonome attempt"), + } satisfies TaxonomyEvaluation["implementation"]["attempts"][number]; + }); + const providerValue = implementation.provider; + const provider = + providerValue === undefined + ? undefined + : (() => { + const candidate = object(providerValue, "Taxonome provider"); + return { + name: string(candidate, "name", "Taxonome provider"), + version: optionalString(candidate, "version"), + origin: string(candidate, "origin", "Taxonome provider"), + }; + })(); + const input = object(evaluation.input, "Taxonome evaluation.input"); + const result = + status === "succeeded" + ? parseAssessment(evaluation.result) + : undefined; + const errorValue = + status === "failed" + ? object(evaluation.error, "Taxonome evaluation.error") + : undefined; + const validationErrors = + errorValue?.validation_errors === undefined + ? undefined + : (() => { + if (!Array.isArray(errorValue.validation_errors)) { + throw new Error("Invalid Taxonome validation errors"); + } + return errorValue.validation_errors.map((candidate) => { + const diagnostic = object(candidate, "Taxonome diagnostic"); + return { + code: string(diagnostic, "code", "Taxonome diagnostic"), + output_path: string( + diagnostic, + "output_path", + "Taxonome diagnostic", + ), + role: optionalString(diagnostic, "role"), + evidence_path: optionalString(diagnostic, "evidence_path"), + message: optionalString(diagnostic, "message"), + }; + }); + })(); + return { + status, + taxonomy: { + id: string(taxonomy, "id", "Taxonome taxonomy"), + version: string(taxonomy, "version", "Taxonome taxonomy"), + digest: string(taxonomy, "digest", "Taxonome taxonomy"), + }, + element: { + id: string(element, "id", "Taxonome element"), + label: string(element, "label", "Taxonome element"), + ...(typeof element.category_id === "string" + ? { category_id: element.category_id } + : {}), + }, + implementation: { + type: implementationType, + handler: optionalString(implementation, "handler"), + provider, + attempts, + }, + input, + ...(result ? { result } : {}), + ...(errorValue + ? { + error: { + code: string(errorValue, "code", "Taxonome evaluation error"), + message: string( + errorValue, + "message", + "Taxonome evaluation error", + ), + ...(validationErrors + ? { validation_errors: validationErrors } + : {}), + }, + } + : {}), + }; + }); + return { + run: { + id: string(run, "id", "Taxonome run"), + scenario: { + id: string(scenario, "id", "Taxonome scenario"), + label: string(scenario, "label", "Taxonome scenario"), + ...(typeof scenario.task === "string" ? { task: scenario.task } : {}), + }, + profile: { + id: string(profile, "id", "Taxonome profile"), + label: string(profile, "label", "Taxonome profile"), + }, + }, + results, + ...(root.metadata !== undefined ? { metadata: root.metadata } : {}), + }; +} diff --git a/apps/portal/src/pages/ExperimentDetail.play.test.tsx b/apps/portal/src/pages/ExperimentDetail.play.test.tsx index 464dba2a0..6818b4c25 100644 --- a/apps/portal/src/pages/ExperimentDetail.play.test.tsx +++ b/apps/portal/src/pages/ExperimentDetail.play.test.tsx @@ -26,6 +26,9 @@ vi.mock("@/lib/api", async (importOriginal) => { listReportTemplateVersions: vi.fn(), updateExperimentReportTemplates: vi.fn(), restoreExperiment: vi.fn(), + getRun: vi.fn(), + listRunAttempts: vi.fn(), + getProfile: vi.fn(), }, }; }); @@ -72,8 +75,25 @@ beforeAll(() => { requestId: "request-current", runId: "run-current", task: "Implement the current benchmark task", - status: "done", - outcome: "success", + status: "processing", + }, + { + requestId: "request-queued", + runId: "run-queued", + task: "Implement the queued benchmark task", + status: "queued", + }, + { + requestId: "request-deleted", + runId: "run-deleted", + task: "Deleted exact member", + status: "deleted", + }, + { + requestId: "request-missing", + runId: "run-missing", + task: "Missing exact member", + status: "missing", }, ], handlerProgress: { "pp-readout": { status: "done" } }, @@ -93,7 +113,7 @@ beforeAll(() => { experimentExecutionId: "execution-history", attemptNumber: 1, readiness: "failed", - requestCount: 1, + requestCount: 4, requests: [ { requestId: "request-history", @@ -114,7 +134,7 @@ beforeAll(() => { revision: { experimentRevisionId: "revision-demo", revisionNumber: 1, - memberCount: 1, + memberCount: 4, reportTemplateIds: ["quality-summary"], }, execution: currentExecution, @@ -125,6 +145,22 @@ beforeAll(() => { }; vi.mocked(api.getExperiment).mockResolvedValue(experiment); + vi.mocked(api.getRun).mockResolvedValue({ + _id: "request-current", + id: "request-current", + scenario: { + task: "Implement the current benchmark task", + criteria: [], + }, + workerType: "coder-acp-copilot", + createdAt: "2026-10-16T09:00:00.000Z", + run: { + _id: "run-current", + attemptNumber: 2, + status: "processing", + }, + }); + vi.mocked(api.listRunAttempts).mockResolvedValue([]); vi.mocked(api.listExperimentExecutions).mockResolvedValue({ items: [currentExecution, historicalExecution], }); @@ -207,7 +243,7 @@ beforeAll(() => { ]); }); -describe("ExperimentDetail stories", () => { +describe("ExperimentDetail stories", { timeout: 30_000 }, () => { it("runs the attempt-selection interaction", async () => { const StoryRender = experimentDetailMeta.render; if (!StoryRender) throw new Error("Experiment detail story is missing render"); diff --git a/apps/portal/src/pages/ExperimentDetail.stories.tsx b/apps/portal/src/pages/ExperimentDetail.stories.tsx index 417ee5d81..3eebf3a7c 100644 --- a/apps/portal/src/pages/ExperimentDetail.stories.tsx +++ b/apps/portal/src/pages/ExperimentDetail.stories.tsx @@ -18,11 +18,42 @@ const currentExecution = { requestId: "request-current", runId: "run-current", task: "Implement the current benchmark task", - status: "done", - outcome: "success", + criteria: ["uses_typescript"], + taskPromptId: "task-prompt-current", + status: "processing", + workerType: "coder-acp-copilot", + model: "gpt-5.6-sol", + reasoningEffort: "high", + agentVersion: "copilot-dev", + os: { platform: "linux", release: "6.8", arch: "arm64" }, + mcpServers: ["github"], + skillRevisions: ["owner/skills/typescript@r1"], + extensions: ["ms-vscode.vscode-typescript-next@1.0.0"], + priority: 5, + attemptNumber: 2, + worker: { instanceId: "worker-current", podName: "pod-current" }, + startedAt: "2026-10-16T09:59:30.000Z", + }, + { + requestId: "request-queued", + runId: "run-queued", + task: "Implement the queued benchmark task", + status: "queued", workerType: "coder-acp-copilot", model: "gpt-5.6-sol", }, + { + requestId: "request-deleted", + runId: "run-deleted", + task: "Deleted exact member", + status: "deleted", + }, + { + requestId: "request-missing", + runId: "run-missing", + task: "Missing exact member", + status: "missing", + }, ], handlerProgress: { "pp-readout": { status: "done" }, @@ -78,7 +109,7 @@ const experiment = { revision: { experimentRevisionId: "revision-demo", revisionNumber: 1, - memberCount: 1, + memberCount: 4, reportTemplateIds: ["quality-summary"], }, execution: currentExecution, @@ -250,6 +281,29 @@ const meta = { }, ]), ), + http.get("/api/v1/requests/request-current", () => + HttpResponse.json({ + _id: "request-current", + id: "request-current", + scenario: { + task: "Implement the current benchmark task", + criteria: ["uses_typescript"], + }, + workerType: "coder-acp-copilot", + model: "gpt-5.6-sol", + agentVersion: "copilot-dev", + createdAt: "2026-10-16T09:00:00.000Z", + run: { + _id: "run-current", + attemptNumber: 2, + status: "processing", + startedAt: "2026-10-16T09:59:30.000Z", + }, + }), + ), + http.get("/api/v1/requests/request-current/runs", () => + HttpResponse.json([]), + ), ], }, }, @@ -280,12 +334,31 @@ export const AttemptSelectionAndRunsTable: Story = { await expect( screen.getAllByText("Implement the current benchmark task")[0], ).toBeVisible(); + const processingBadge = screen + .getAllByText("Processing") + .find((element) => element.className.includes("animate-shimmer")); + if (!processingBadge) throw new Error("Processing status is not animated"); + await expect(processingBadge).toBeVisible(); + await expect(screen.getAllByText("Deleted")[0]).toBeVisible(); + await expect(screen.getAllByText("Missing")[0]).toBeVisible(); const embeddedTable = canvasElement.querySelector( '[data-slot="embedded-table-block"]', ); if (!embeddedTable) throw new Error("Embedded runs table block is missing"); await expect(embeddedTable).toHaveClass("space-y-3"); await expect(embeddedTable.parentElement).toHaveClass("p-6"); + const currentTask = screen.getAllByText( + "Implement the current benchmark task", + )[0]; + const currentRow = currentTask.closest("tr"); + if (!currentRow) throw new Error("Current run row is missing"); + await userEvent.click(currentRow); + await expect( + await screen.findByRole("complementary", { name: "Detail panel" }), + ).toBeVisible(); + await userEvent.click( + screen.getByRole("button", { name: "Close detail panel" }), + ); await userEvent.click( canvas.getByRole("combobox", { name: "Experiment attempt" }), diff --git a/apps/portal/src/pages/ExperimentDetail.test.tsx b/apps/portal/src/pages/ExperimentDetail.test.tsx index f6d40a06c..f17c9d011 100644 --- a/apps/portal/src/pages/ExperimentDetail.test.tsx +++ b/apps/portal/src/pages/ExperimentDetail.test.tsx @@ -40,6 +40,10 @@ vi.mock("@/lib/api", async (importOriginal) => { updateExperimentRequests: vi.fn(), deleteExperiment: vi.fn(), restoreExperiment: vi.fn(), + getRun: vi.fn(), + listRunAttempts: vi.fn(), + getProfile: vi.fn(), + listAgents: vi.fn(async () => []), experimentReportStructuredUrl: vi.fn( (_experimentId: string, executionId: string, reportId: string) => `/api/v1/experiments/experiment-1/executions/${executionId}/reports/${reportId}/structured?projectId=project-1`, @@ -67,10 +71,48 @@ const currentRequests = Array.from({ length: 11 }, (_, index) => ({ requestId: `request-${String(index + 1).padStart(2, "0")}`, runId: `run-current-${String(index + 1).padStart(2, "0")}`, task: `Current task ${index + 1}`, - status: index === 0 ? "done" : "pending", - outcome: index === 0 ? "success" : undefined, + status: + index === 0 + ? "done" + : index === 1 + ? "processing" + : index === 2 + ? "queued" + : index === 3 + ? "deleted" + : index === 4 + ? "missing" + : "pending", + outcome: index === 0 ? "succeeded" : undefined, + criteria: ["uses_python"], + taskPromptId: "task-prompt-1", workerType: "coder-acp-copilot", model: "gpt-5.6-sol", + reasoningEffort: "high", + agentVersion: "copilot-dev", + os: { platform: "linux", release: "6.8", arch: "arm64" }, + mcpServers: ["github"], + skillRevisions: ["owner/skills/python@r1"], + extensions: ["ms-python.python@1.0.0"], + profileId: "profile-1", + priority: 5, + attemptNumber: 1, + taxonomyHandler: + index === 0 + ? { status: "done" as const, resultAvailable: true } + : index === 1 + ? { status: "processing" as const, resultAvailable: false } + : index === 2 + ? { status: "queued" as const, resultAvailable: false } + : index === 3 + ? { + status: "failed" as const, + resultAvailable: false, + error: "Taxonome execution failed", + } + : index === 4 + ? { status: "pending" as const, resultAvailable: false } + : undefined, })); const currentExecution: ExperimentExecutionView = { @@ -179,7 +221,12 @@ const qualityReport: Report = { function LocationProbe() { const location = useLocation(); - return
{location.pathname}{location.search}
; + return ( + <> +
{location.pathname}{location.search}
+
{JSON.stringify(location.state)}
+ + ); } function renderDetail( @@ -238,6 +285,47 @@ beforeEach(() => { vi.mocked(api.listExperimentReports).mockResolvedValue([]); vi.mocked(api.deleteExperiment).mockReset(); vi.mocked(api.restoreExperiment).mockReset(); + vi.mocked(api.getRun).mockImplementation(async (requestId) => { + const request = + currentRequests.find((candidate) => candidate.requestId === requestId) ?? + historicalExecution.requests[0]; + return { + _id: request.requestId, + id: request.requestId, + scenario: { + task: request.task ?? "", + criteria: request.criteria ?? [], + }, + workerType: request.workerType ?? "coder-acp-copilot", + model: request.model, + agentVersion: request.agentVersion, + createdAt: "2026-10-16T10:00:00.000Z", + run: { + _id: request.runId, + attemptNumber: request.attemptNumber ?? 1, + status: + request.status === "processing" + ? "processing" + : request.status === "queued" + ? "queued" + : "done", + outcome: request.outcome === "failed" ? "failed" : "succeeded", + }, + }; + }); + vi.mocked(api.listRunAttempts).mockResolvedValue([]); + vi.mocked(api.getProfile).mockResolvedValue(undefined as never); + vi.mocked(api.listAgents).mockResolvedValue([ + { + _id: "coder-acp-copilot", + name: "GitHub Copilot CLI", + supportedModels: ["gpt-5.6-sol"], + available: true, + capabilities: {}, + versions: [], + createdAt: "2026-10-16T08:00:00.000Z", + }, + ]); }); afterEach(() => { @@ -246,7 +334,7 @@ afterEach(() => { vi.clearAllMocks(); }); -describe("ExperimentDetail", { timeout: 10_000 }, () => { +describe("ExperimentDetail", { timeout: 30_000 }, () => { it("starts an asynchronous deletion from the confirmation dialog", async () => { vi.mocked(api.deleteExperiment).mockResolvedValue({ operationId: "operation-delete", @@ -345,6 +433,17 @@ describe("ExperimentDetail", { timeout: 10_000 }, () => { expect(await screen.findByText("Runs in attempt 2")).toBeTruthy(); expect(screen.getAllByText("Current task 1").length).toBeGreaterThan(0); + expect( + screen + .getAllByText("Processing") + .some((element) => element.className.includes("animate-shimmer")), + ).toBe(true); + expect(screen.getAllByText("Queued").length).toBeGreaterThan(0); + expect(screen.getAllByText("Deleted").length).toBeGreaterThan(0); + expect(screen.getAllByText("Missing").length).toBeGreaterThan(0); + expect(screen.getAllByText("Completed").length).toBeGreaterThan(0); + expect(screen.getAllByText("Pending").length).toBeGreaterThan(0); + expect(screen.getAllByText("Processing").length).toBeGreaterThan(0); expect(screen.queryAllByText("Historical task")).toHaveLength(0); expect(screen.getByText("1–10 of 11 runs")).toBeTruthy(); const embeddedTable = document.querySelector( @@ -388,13 +487,46 @@ describe("ExperimentDetail", { timeout: 10_000 }, () => { expect( screen.queryByRole("button", { name: "Retry processing" }), ).toBeNull(); - fireEvent.click(screen.getAllByText("Historical task")[0]); + fireEvent.click(screen.getAllByRole("button", { name: "View" })[0]); expect((await screen.findByTestId("location")).textContent).toBe( "/runs/request-history?runId=run-history", ); + expect(screen.getByTestId("location-state").textContent).toContain( + "Back to experiment", + ); }); + it("opens, switches, and closes the shared Run preview without navigating", async () => { + renderDetail(); + + const firstTask = (await screen.findAllByText("Current task 1"))[0]; + const firstRow = firstTask.closest("tr"); + if (!firstRow) throw new Error("Run row is missing"); + fireEvent.click(firstRow); + expect( + await screen.findByRole("complementary", { name: "Detail panel" }), + ).toBeTruthy(); + expect(screen.getByTestId("location").textContent).toBe( + "/experiments/experiment-1", + ); + + fireEvent.click(screen.getByRole("button", { name: "Close detail panel" })); + await waitFor(() => { + expect( + screen.queryByRole("complementary", { name: "Detail panel" }), + ).toBeNull(); + }); + + const secondTask = (await screen.findAllByText("Current task 2"))[0]; + const secondRow = secondTask.closest("tr"); + if (!secondRow) throw new Error("Second run row is missing"); + fireEvent.click(secondRow); + expect( + await screen.findByRole("complementary", { name: "Detail panel" }), + ).toBeTruthy(); + }); + it("keeps processing retry disabled for a historical attempt", async () => { renderDetail( "/experiments/experiment-1?executionId=execution-history&tab=processing", @@ -472,7 +604,7 @@ describe("ExperimentDetail", { timeout: 10_000 }, () => { .mockRejectedValueOnce(new Error("Report service unavailable")) .mockResolvedValueOnce([]); - renderDetail(); + renderDetail("/experiments/experiment-1?tab=reports"); expect(await screen.findByText("Report service unavailable")).toBeTruthy(); fireEvent.click(screen.getByRole("button", { name: /^Retry$/ })); @@ -485,7 +617,7 @@ describe("ExperimentDetail", { timeout: 10_000 }, () => { { includeDeleted: false }, ); expect( - await screen.findByText("Runs in attempt 2"), + await screen.findByText("No report available"), ).toBeTruthy(); }); diff --git a/apps/portal/src/pages/ExperimentDetail.tsx b/apps/portal/src/pages/ExperimentDetail.tsx index cf2d9cf3c..3ca908e31 100644 --- a/apps/portal/src/pages/ExperimentDetail.tsx +++ b/apps/portal/src/pages/ExperimentDetail.tsx @@ -7,12 +7,15 @@ import { ArchiveRestore, ArrowLeft, Clock, + Download, + Eye, RotateCcw, Trash2, Unlink, } from "lucide-react"; import { Link, + useLocation, useNavigate, useParams, useSearchParams, @@ -31,14 +34,34 @@ import { import { CustomizeColumnsLink, CustomizeColumnsPanel, - DataTable, EmbeddedTableBlock, - Pagination, useColumnOrder, useHiddenColumns, type CustomizeColumnsOption, type DataTableColumn, } from "@/components/list-layout"; +import { + RunAttemptCell, + RunCriteriaCell, + RunEffortCell, + RunExtensionsCell, + RunIdCell, + RunMcpCell, + RunModelCell, + RunOsCell, + RunOutcomeCell, + RunPriorityCell, + RunProfileCell, + RunSkillsCell, + RunStatusCell, + RunTaskCell, + RunTaxonomyCell, + RunVersionCell, + RunWorkerCell, + RunsTable, + type RunTableItem, +} from "@/components/runs-table"; +import { RunPreviewPanel } from "./RunPreviewPanel"; import { Select, SelectContent, @@ -63,17 +86,30 @@ import type { ExperimentView, } from "@/lib/experiments"; import { experimentDisplayStatus } from "@/lib/experiments"; -import { formatDate, truncate } from "@/lib/utils"; +import { formatDate } from "@/lib/utils"; import type { ReportTemplate } from "@/types"; +import { useFeatureFlags } from "@/contexts/FeatureFlagContext"; +import { runNavigationState } from "@/lib/run-navigation"; +import type { RunOutcome } from "@/types"; const RUN_COLUMN_OPTIONS: CustomizeColumnsOption[] = [ { id: "requestId", label: "ID", required: true }, { id: "status", label: "Status" }, { id: "outcome", label: "Outcome" }, + { id: "taxonomy", label: "Taxonomy" }, { id: "task", label: "Task" }, + { id: "criteria", label: "Criteria" }, { id: "worker", label: "Worker" }, { id: "model", label: "Model" }, + { id: "effort", label: "Effort" }, + { id: "version", label: "Version" }, + { id: "os", label: "OS" }, + { id: "mcp", label: "MCP" }, + { id: "skills", label: "Skills" }, + { id: "extensions", label: "Extensions" }, { id: "profile", label: "Profile" }, + { id: "priority", label: "Priority" }, + { id: "attempt", label: "Attempt" }, { id: "runId", label: "Run ID" }, ]; const RUN_COLUMN_IDS = RUN_COLUMN_OPTIONS.map((column) => column.id); @@ -124,21 +160,114 @@ function requestSortValue( return request.runId; case "task": return request.task ?? ""; + case "criteria": + return request.criteria?.join("\u0000") ?? ""; case "status": return request.status ?? ""; case "outcome": return request.outcome ?? ""; + case "taxonomy": + return request.taxonomyHandler?.status ?? ""; case "worker": return request.workerType ?? ""; case "model": return request.model ?? ""; + case "effort": + return request.reasoningEffort ?? ""; + case "version": + return request.agentVersion ?? ""; + case "os": + return request.os?.platform ?? ""; + case "mcp": + return request.mcpServers?.join("\u0000") ?? ""; + case "skills": + return request.skillRevisions?.join("\u0000") ?? ""; + case "extensions": + return request.extensions?.join("\u0000") ?? ""; case "profile": return request.profileId ?? ""; + case "priority": + return String(request.priority ?? ""); + case "attempt": + return String(request.attemptNumber ?? ""); default: return ""; } } +function experimentRunStatus( + status: string | undefined, + ): RunTableItem["status"] { + if (!status) return undefined; + const normalized = status.toLowerCase(); + if ( + normalized === "pending" || + normalized === "queued" || + normalized === "processing" || + normalized === "paused" || + normalized === "done" || + normalized === "completed" || + normalized === "failed" || + normalized === "cancelled" || + normalized === "deleted" || + normalized === "missing" + ) { + return normalized; + } + return undefined; + } + +function experimentRunOutcome( + outcome: string | undefined, + ): RunOutcome | undefined { + if (outcome === "success") return "succeeded"; + if (outcome === "failure") return "failed"; + if ( + outcome === "succeeded" || + outcome === "failed" || + outcome === "finished" + ) { + return outcome; + } + return undefined; + } + +function experimentRunTableItem( + request: ExperimentRequestSummaryView, + ): RunTableItem { + return { + requestId: request.requestId, + runId: request.runId, + task: request.task, + criteria: request.criteria, + taskPromptId: request.taskPromptId, + workerType: request.workerType, + model: request.model, + reasoningEffort: request.reasoningEffort, + agentVersion: request.agentVersion, + os: request.os, + mcpServers: request.mcpServers, + skillRevisions: request.skillRevisions, + extensions: request.extensions, + profileId: request.profileId, + priority: request.priority, + status: experimentRunStatus(request.status), + outcome: experimentRunOutcome(request.outcome), + attemptNumber: request.attemptNumber, + worker: request.worker, + lastHeartbeatAt: request.lastHeartbeatAt, + startedAt: request.startedAt, + taxonomyHandler: request.taxonomyHandler, + }; +} + +function isUnavailableExperimentMember( + request: ExperimentRequestSummaryView, +): boolean { + const status = experimentRunStatus(request.status); + return status === "missing" || status === "deleted"; +} + function ExecutionSummary({ execution, current, @@ -278,7 +407,10 @@ function TemplateCategory({ export function ExperimentDetail() { const { id } = useParams<{ id: string }>(); const navigate = useNavigate(); + const location = useLocation(); const queryClient = useQueryClient(); + const { isFeatureEnabledFailClosed } = useFeatureFlags(); + const taxonomeEnabled = isFeatureEnabledFailClosed("taxonome"); const [searchParams, setSearchParams] = useSearchParams(); const [selectedRequestIds, setSelectedRequestIds] = useState>(new Set()); const [removeDialogOpen, setRemoveDialogOpen] = useState(false); @@ -298,6 +430,8 @@ export function ExperimentDetail() { useState("requestId"); const [sortDirection, setSortDirection] = useState("asc"); + const [previewMember, setPreviewMember] = + useState(null); const columnVisibility = useHiddenColumns({ storageKey: "experiment-runs", defaultHidden: ["profile", "runId"], @@ -374,6 +508,23 @@ export function ExperimentDetail() { return 5_000; }, }); + const taxonomyReadoutQuery = useQuery({ + queryKey: ["experiment-taxonomy-readout", id], + queryFn: () => api.getTaxonomyReadout(id!), + enabled: Boolean(id && taxonomeEnabled && !isViewingHistorical), + retry: false, + }); + const retryTaxonomyReadout = useMutation({ + mutationFn: () => api.retryTaxonomyReadout(id!), + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: ["experiment-taxonomy-readout", id], + }); + toast.success("Taxonome readout generated"); + }, + onError: (error) => + toast.error(error instanceof Error ? error.message : "Readout failed"), + }); const retry = useMutation({ mutationFn: () => api.retryExperiment( @@ -680,6 +831,7 @@ export function ExperimentDetail() { useEffect(() => { setSelectedRequestIds(new Set()); setRemoveDialogOpen(false); + setPreviewMember(null); }, [activeExecution?.experimentExecutionId]); const pagedRequests = useMemo(() => { @@ -688,6 +840,12 @@ export function ExperimentDetail() { }, [page, pageSize, sortedRequests]); const columns = useMemo[]>(() => { + const item = (request: ExperimentRequestSummaryView) => + experimentRunTableItem(request); + const returnState = runNavigationState( + `${location.pathname}${location.search}`, + "Back to experiment", + ); const naturalColumns: DataTableColumn[] = [ { id: "requestId", @@ -695,9 +853,7 @@ export function ExperimentDetail() { sortable: true, width: "120px", sticky: "left", - cell: (request) => ( - - ), + cell: (request) => , }, { id: "status", @@ -705,12 +861,7 @@ export function ExperimentDetail() { sortable: true, width: "110px", hidden: columnVisibility.isHidden("status"), - cell: (request) => - request.status ? ( - {request.status} - ) : ( - — - ), + cell: (request) => , }, { id: "outcome", @@ -718,23 +869,28 @@ export function ExperimentDetail() { sortable: true, width: "110px", hidden: columnVisibility.isHidden("outcome"), - cell: (request) => - request.outcome ? ( - {request.outcome} - ) : ( - — - ), + cell: (request) => , + }, + { + id: "taxonomy", + header: "Taxonomy", + width: "150px", + hidden: columnVisibility.isHidden("taxonomy"), + cell: (request) => , }, { id: "task", header: "Task", sortable: true, hidden: columnVisibility.isHidden("task"), - cell: (request) => ( - - {request.task ? truncate(request.task, 80) : "—"} - - ), + cell: (request) => , + }, + { + id: "criteria", + header: "Criteria", + width: "180px", + hidden: columnVisibility.isHidden("criteria"), + cell: (request) => , }, { id: "worker", @@ -742,11 +898,7 @@ export function ExperimentDetail() { sortable: true, width: "180px", hidden: columnVisibility.isHidden("worker"), - cell: (request) => ( - - {request.workerType ?? "—"} - - ), + cell: (request) => , }, { id: "model", @@ -754,9 +906,49 @@ export function ExperimentDetail() { sortable: true, width: "180px", hidden: columnVisibility.isHidden("model"), - cell: (request) => ( - {request.model ?? "—"} - ), + cell: (request) => , + }, + { + id: "effort", + header: "Effort", + width: "100px", + hidden: columnVisibility.isHidden("effort"), + cell: (request) => , + }, + { + id: "version", + header: "Version", + width: "110px", + hidden: columnVisibility.isHidden("version"), + cell: (request) => , + }, + { + id: "os", + header: "OS", + width: "120px", + hidden: columnVisibility.isHidden("os"), + cell: (request) => , + }, + { + id: "mcp", + header: "MCP", + width: "180px", + hidden: columnVisibility.isHidden("mcp"), + cell: (request) => , + }, + { + id: "skills", + header: "Skills", + width: "180px", + hidden: columnVisibility.isHidden("skills"), + cell: (request) => , + }, + { + id: "extensions", + header: "Extensions", + width: "190px", + hidden: columnVisibility.isHidden("extensions"), + cell: (request) => , }, { id: "profile", @@ -764,11 +956,22 @@ export function ExperimentDetail() { sortable: true, width: "150px", hidden: columnVisibility.isHidden("profile"), - cell: (request) => ( - - {request.profileId ?? "—"} - - ), + cell: (request) => , + }, + { + id: "priority", + header: "Priority", + sortable: true, + width: "80px", + hidden: columnVisibility.isHidden("priority"), + cell: (request) => , + }, + { + id: "attempt", + header: "Attempt", + width: "80px", + hidden: columnVisibility.isHidden("attempt"), + cell: (request) => , }, { id: "runId", @@ -780,7 +983,7 @@ export function ExperimentDetail() { }, ]; const byId = new Map(naturalColumns.map((column) => [column.id, column])); - return columnOrder.order + const ordered = columnOrder.order .map((columnId) => byId.get(columnId)) .filter( ( @@ -788,7 +991,65 @@ export function ExperimentDetail() { ): column is DataTableColumn => Boolean(column), ); - }, [columnOrder.order, columnVisibility]); + return [ + ...ordered, + { + id: "actions", + header: "", + width: "80px", + sticky: "right", + cell: (request) => { + const unavailable = isUnavailableExperimentMember(request); + return ( +
+ + +
+ ); + }, + }, + ]; + }, [ + columnOrder.order, + columnVisibility, + location.pathname, + location.search, + navigate, + ]); const selectExecution = (executionId: string) => { const next = new URLSearchParams(searchParams); @@ -1109,16 +1370,24 @@ export function ExperimentDetail() { onReorder={columnOrder.setOrder} /> )} -
- + request.requestId} - onRowClick={(request) => - navigate( - `/runs/${request.requestId}?runId=${encodeURIComponent(request.runId)}`, - ) - } + activeId={previewMember?.requestId} + onRowClick={(request) => { + if (isUnavailableExperimentMember(request)) { + return; + } + setPreviewMember(request); + }} sort={sortColumn} sortDir={sortDirection} onSortChange={changeSort} @@ -1131,6 +1400,16 @@ export function ExperimentDetail() { onToggle: (requestId) => setSelectedRequestIds((current) => { const id = String(requestId); + const request = activeExecution.requests.find( + (candidate) => + candidate.requestId === id, + ); + if ( + !request || + isUnavailableExperimentMember(request) + ) { + return current; + } const next = new Set(current); if (next.has(id)) next.delete(id); else next.add(id); @@ -1139,7 +1418,19 @@ export function ExperimentDetail() { onToggleAll: (requestIds) => setSelectedRequestIds((current) => { const next = new Set(current); - const ids = requestIds.map(String); + const eligible = new Set( + activeExecution.requests + .filter( + (request) => + !isUnavailableExperimentMember( + request, + ), + ) + .map((request) => request.requestId), + ); + const ids = requestIds + .map(String) + .filter((id) => eligible.has(id)); const allSelected = ids.every((requestId) => next.has(requestId) ); @@ -1158,18 +1449,31 @@ export function ExperimentDetail() { : `${activeExecution.requestCount} linked runs are stored outside the resolved response.`}
} + pagination={ + sortedRequests.length > 0 + ? { + page, + pageSize, + total: sortedRequests.length, + pageSizeOptions: [10, 25, 50], + onPageChange: setPage, + onPageSizeChange: setPageSize, + } + : undefined + } /> - {sortedRequests.length > 0 && ( - + {previewMember && ( +
+ setPreviewMember(null)} + navigationState={runNavigationState( + `${location.pathname}${location.search}`, + "Back to experiment", + )} + /> +
)} @@ -1178,7 +1482,55 @@ export function ExperimentDetail() { - + + {taxonomeEnabled && !isViewingHistorical && ( + + + + Taxonome readout + + + + + {taxonomyReadoutQuery.data ? ( + <> + + {taxonomyReadoutQuery.data.status} + + {taxonomyReadoutQuery.data.error && ( +

{taxonomyReadoutQuery.data.error}

+ )} + {taxonomyReadoutQuery.data.status === "done" && ( + + )} + + ) : ( +

+ Generate a typed readout after exact-run taxonomy evaluations finish. +

+ )} +
+
+ )} { + const original = await importOriginal(); + return { + ...original, + api: { + ...original.api, + getRun: vi.fn(async () => { + throw new Error("Run not found"); + }), + listRunAttempts: vi.fn(async () => []), + }, + }; +}); + +import { RunDetail } from "./RunDetail"; + +function renderDetail(state?: unknown) { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render( + + + + } /> + + + , + ); +} + +describe("RunDetail return navigation", () => { + afterEach(cleanup); + + it("returns to the exact Experiment origin", async () => { + renderDetail( + runNavigationState( + "/experiments/experiment-1?tab=runs&executionId=execution-2", + "Back to experiment", + ), + ); + + expect( + ( + await screen.findByRole("link", { name: "Back to experiment" }) + ).getAttribute("href"), + ).toBe("/experiments/experiment-1?tab=runs&executionId=execution-2"); + }); + + it("returns to the exact Runs query state", async () => { + renderDetail( + runNavigationState( + "/runs?groupBy=experimentId&page=2", + "Back to runs", + ), + ); + + expect( + ( + await screen.findByRole("link", { name: "Back to runs" }) + ).getAttribute("href"), + ).toBe("/runs?groupBy=experimentId&page=2"); + }); + + it.each([ + undefined, + { returnTo: "https://example.com", returnLabel: "External" }, + ])("falls back to Runs for direct or invalid state %#", async (state) => { + renderDetail(state); + + expect( + ( + await screen.findByRole("link", { name: "Back to runs" }) + ).getAttribute("href"), + ).toBe("/runs"); + }); +}); diff --git a/apps/portal/src/pages/RunDetail.taxonomy.test.tsx b/apps/portal/src/pages/RunDetail.taxonomy.test.tsx new file mode 100644 index 000000000..d4013306a --- /dev/null +++ b/apps/portal/src/pages/RunDetail.taxonomy.test.tsx @@ -0,0 +1,174 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { cleanup, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import { runNavigationState } from "@/lib/run-navigation"; + +const mocks = vi.hoisted(() => ({ + getRun: vi.fn(), + listRunAttempts: vi.fn(), + getTaxonomyEvaluationJson: vi.fn(), + getRunReports: vi.fn(), + listReportTemplates: vi.fn(), +})); + +vi.mock("@/lib/api", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + api: { + ...original.api, + getRun: mocks.getRun, + listRunAttempts: mocks.listRunAttempts, + getTaxonomyEvaluationJson: mocks.getTaxonomyEvaluationJson, + getRunReports: mocks.getRunReports, + listReportTemplates: mocks.listReportTemplates, + }, + }; +}); +vi.mock("@/hooks/use-log-stream", () => ({ + useLogStream: () => ({ + logs: [], + isConnected: false, + isDone: true, + error: null, + }), +})); + +import { RunDetail } from "./RunDetail"; + +const taxonomyResult = { + run: { + id: "run-1", + scenario: { id: "scenario-1", label: "Scenario" }, + profile: { id: "profile-1", label: "Profile" }, + }, + results: [ + { + status: "succeeded", + taxonomy: { + id: "ax-taxonomy", + version: "1.0.0", + digest: "a".repeat(64), + }, + element: { id: "element-1", label: "Observed behavior" }, + implementation: { + type: "deterministic", + handler: "element-1", + attempts: [{ attempt: 1, kind: "initial", outcome: "succeeded" }], + }, + input: { sha256: "b".repeat(64) }, + result: { state: "completed", observed: false }, + }, + ], +}; + +function runDocument(resultAvailable = true) { + return { + _id: "request-1", + scenario: { task: "Test task", criteria: [] }, + workerType: "coder-acp-copilot", + model: "model", + createdAt: "2026-10-29T12:00:00.000Z", + run: { + _id: "run-1", + status: "done", + handlerStatus: { + "pp-taxonomy": { + status: "done", + version: "1", + updatedAt: "2026-10-29T12:01:00.000Z", + claimToken: "claim-1", + }, + }, + ...(resultAvailable + ? { + taxonomyEvaluation: { + url: "https://storage.example/result.json.gz", + schemaVersion: "2", + contentSha256: "c".repeat(64), + sizeBytes: 123, + }, + } + : {}), + }, + }; +} + +function renderDetail(options?: { path?: string; state?: unknown }) { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render( + + + + } /> + + + , + ); +} + +describe("RunDetail Taxonomy tab", () => { + beforeEach(() => { + mocks.getRun.mockResolvedValue(runDocument()); + mocks.listRunAttempts.mockResolvedValue([]); + mocks.getTaxonomyEvaluationJson.mockResolvedValue(taxonomyResult); + mocks.getRunReports.mockResolvedValue([]); + mocks.listReportTemplates.mockResolvedValue([]); + }); + + afterEach(() => { + cleanup(); + vi.clearAllMocks(); + }); + + it("loads and renders the validated typed result for the exact run", async () => { + const user = userEvent.setup(); + renderDetail({ + path: "/runs/request-1", + state: runNavigationState( + "/experiments/experiment-1?tab=runs&executionId=execution-1", + "Back to experiment", + ), + }); + await user.click(await screen.findByRole("tab", { name: "Taxonomy" })); + + expect(await screen.findByText("Taxonome result")).toBeTruthy(); + expect(screen.getByText("Observed behavior")).toBeTruthy(); + expect(screen.getByText("Completed")).toBeTruthy(); + expect(mocks.getTaxonomyEvaluationJson).toHaveBeenCalledWith( + "request-1", + "run-1", + ); + expect(screen.getByRole("link", { name: "Download JSON" })).toBeTruthy(); + expect( + screen.getByRole("link", { name: "Back to experiment" }).getAttribute( + "href", + ), + ).toBe( + "/experiments/experiment-1?tab=runs&executionId=execution-1", + ); + }); + + it("shows result unavailable without fetching a blob when no reference exists", async () => { + mocks.getRun.mockResolvedValue(runDocument(false)); + renderDetail(); + + expect(await screen.findByText("Result unavailable")).toBeTruthy(); + expect(mocks.getTaxonomyEvaluationJson).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/portal/src/pages/RunDetail.tsx b/apps/portal/src/pages/RunDetail.tsx index c0d483adc..4f1b88286 100644 --- a/apps/portal/src/pages/RunDetail.tsx +++ b/apps/portal/src/pages/RunDetail.tsx @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { useParams, Link, useNavigate, useSearchParams } from "react-router-dom"; +import { useParams, Link, useLocation, useNavigate, useSearchParams } from "react-router-dom"; import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"; import { api } from "@/lib/api"; import { RetryConfirmDialog } from "@/components/RetryConfirmDialog"; @@ -17,6 +17,8 @@ import { import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { StatusBadge, OutcomeBadge } from "@/components/StatusBadge"; import { EnrichmentBadge } from "@/components/EnrichmentBadge"; +import { TaxonomyStatus } from "@/components/TaxonomyStatus"; +import { TaxonomeResultView } from "@/components/TaxonomeResultView"; import { ReportStatusBadge } from "@/components/ReportStatusBadge"; import { LogViewer } from "@/components/LogViewer"; import { TurnTimeline } from "@/components/TurnTimeline"; @@ -58,6 +60,7 @@ import { useShiftModifier } from "@/hooks/useShiftModifier"; import { getRetryButtonState } from "@/components/RetryButton"; import { deriveEnrichmentStatus } from "@/lib/enrichment"; import { getRunSkillReferences } from "@/lib/skill-spec"; +import { resolveRunReturnState } from "@/lib/run-navigation"; /** A compact labeled stat: a micro uppercase label above its value. */ function MetaItem({ label, value, title }: { label: string; value: ReactNode; title?: string }) { @@ -421,14 +424,11 @@ function WholeRunObservationsPanel({ export function RunDetail() { const { id, tab } = useParams<{ id: string; tab?: string }>(); const navigate = useNavigate(); + const location = useLocation(); const queryClient = useQueryClient(); const [searchParams] = useSearchParams(); const selectedRunId = searchParams.get("runId"); - const fromExperimentId = searchParams.get("fromExperiment"); - const fromExperimentExecutionId = searchParams.get("experimentExecution"); - const experimentHref = fromExperimentId - ? `/experiments/${fromExperimentId}${fromExperimentExecutionId ? `?execution=${encodeURIComponent(fromExperimentExecutionId)}` : ""}` - : undefined; + const runReturn = resolveRunReturnState(location.state); const [copied, setCopied] = useState(false); const [reportsView, setReportsView] = useState<"grid" | "list">("grid"); const [reportsFilter, setReportsFilter] = useState<"latest" | "all">("latest"); @@ -489,9 +489,12 @@ export function RunDetail() { const normalizedTab = tab === "observations" ? "taxonomy" : tab; useEffect(() => { if (tab === "observations") { - navigate(`/runs/${id}/taxonomy${selectedRunId ? `?runId=${selectedRunId}` : ""}`, { replace: true }); + navigate( + `/runs/${id}/taxonomy${selectedRunId ? `?runId=${selectedRunId}` : ""}`, + { replace: true, state: location.state }, + ); } - }, [tab, id, selectedRunId, navigate]); + }, [tab, id, selectedRunId, navigate, location.state]); const isActive = activeRun?.status === "pending" || activeRun?.status === "processing"; // Note: "done" is terminal — not active, no log streaming needed @@ -535,6 +538,21 @@ export function RunDetail() { (activeRun?.turns ?? []).some((turn) => (turn.observationResults?.length ?? 0) > 0) || (activeRun?.observationResults?.length ?? 0) > 0; const hasConfiguredObservations = (run?.observations?.length ?? 0) > 0; + const taxonomyEvaluation = activeRun?.taxonomyEvaluation; + const taxonomyHandlerStatus = activeRun?.handlerStatus?.["pp-taxonomy"]; + const hasTaxonomyEvaluation = Boolean(taxonomyEvaluation); + const hasTaxonomyHandler = Boolean(taxonomyHandlerStatus); + const taxonomyResultQuery = useQuery({ + queryKey: ["taxonomy-evaluation-json", id, activeRun?._id], + queryFn: () => api.getTaxonomyEvaluationJson(id!, activeRun!._id), + enabled: Boolean( + id && + activeRun?._id && + activeRun.taxonomyEvaluation && + normalizedTab === "taxonomy", + ), + retry: false, + }); // Aggregate enrichment status across the whole post-processing DAG so the // "Enriched" badge waits for pp-observations instead of flipping after pp-atif. const enrichmentStatus = useMemo( @@ -710,10 +728,10 @@ export function RunDetail() { if (error || !run) { return (
- +
@@ -810,14 +828,10 @@ export function RunDetail() {
{/* Back link + header */}
- + @@ -1021,9 +1035,15 @@ export function RunDetail() { onClick={() => { setShowAttempts(false); if (isCurrent) { - navigate(`/runs/${id}/${tab ?? ""}`, { replace: true }); + navigate(`/runs/${id}/${tab ?? ""}`, { + replace: true, + state: location.state, + }); } else { - navigate(`/runs/${id}/${tab ?? ""}?runId=${attempt._id}`, { replace: true }); + navigate( + `/runs/${id}/${tab ?? ""}?runId=${attempt._id}`, + { replace: true, state: location.state }, + ); } }} > @@ -1155,7 +1175,12 @@ export function RunDetail() { Viewing attempt #{activeRun?.attemptNumber} (historical).{" "} @@ -1166,7 +1191,12 @@ export function RunDetail() { {/* Tabs */} 0 ? "turns" : "logs")} - onValueChange={(value) => navigate(`/runs/${id}/${value}${selectedRunId ? `?runId=${selectedRunId}` : ""}`)} + onValueChange={(value) => + navigate( + `/runs/${id}/${value}${selectedRunId ? `?runId=${selectedRunId}` : ""}`, + { state: location.state }, + ) + } > @@ -1178,7 +1208,7 @@ export function RunDetail() { {hasHarData && Network} {hasHarData && Tool Calls} {hasVideoData && } - {(hasConfiguredObservations || hasObservationResults) && ( + {(hasConfiguredObservations || hasObservationResults || hasTaxonomyHandler || hasTaxonomyEvaluation) && ( Taxonomy {hasObservationResults ? `(${((activeRun?.turns?.reduce((count, turn) => count + (turn.observationResults?.length ?? 0), 0) ?? 0) + (activeRun?.observationResults?.length ?? 0))})` : ""} @@ -1278,8 +1308,62 @@ export function RunDetail() { )} - {(hasConfiguredObservations || hasObservationResults) && ( + {(hasConfiguredObservations || hasObservationResults || hasTaxonomyHandler || hasTaxonomyEvaluation) && ( + {(hasTaxonomyHandler || taxonomyEvaluation) && ( +
+
+

Taxonome evaluation

+

+ Full SDK result for this exact run attempt. +

+
+
+ + {taxonomyEvaluation && ( + <> + {taxonomyEvaluation.schemaVersion} + {taxonomyEvaluation.sizeBytes.toLocaleString()} bytes + {taxonomyEvaluation.contentSha256.slice(0, 12)} + + + )} +
+
+ )} + {activeRun?.taxonomyEvaluation && ( + + )}

Observations

diff --git a/apps/portal/src/pages/RunPreviewPanel.tsx b/apps/portal/src/pages/RunPreviewPanel.tsx index 1a6bacee2..0e8ac53e1 100644 --- a/apps/portal/src/pages/RunPreviewPanel.tsx +++ b/apps/portal/src/pages/RunPreviewPanel.tsx @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { useParams, useNavigate, Link } from "react-router-dom"; +import { useParams, useLocation, useNavigate, Link } from "react-router-dom"; import { useQuery } from "@tanstack/react-query"; import { ExternalLink, FileText } from "lucide-react"; import { api } from "@/lib/api"; @@ -14,13 +14,29 @@ import { TaskPromptBadge } from "@/components/TaskPromptBadge"; import { AgentBadge, agentDisplayName, useAgentCatalog } from "@/components/AgentBadge"; import { GATE_METADATA } from "@/lib/gates"; import { formatDate, formatId, formatDuration, truncate } from "@/lib/utils"; +import type { RunNavigationState } from "@/lib/run-navigation"; -export function RunPreviewPanel() { - const { id } = useParams<{ id: string }>(); +export function RunPreviewPanel({ + requestId, + runId, + onClose, + navigationState, +}: { + requestId?: string; + runId?: string; + onClose?: () => void; + navigationState?: RunNavigationState; +} = {}) { + const { id: routeId } = useParams<{ id: string }>(); + const id = requestId ?? routeId; const navigate = useNavigate(); + const location = useLocation(); + const effectiveNavigationState = + navigationState ?? + (location.state as RunNavigationState | undefined); const { agentById } = useAgentCatalog(); - const { data: run, isLoading, error } = useQuery({ + const { data: request, isLoading, error } = useQuery({ queryKey: ["run", id], queryFn: () => api.getRun(id!), enabled: !!id, @@ -30,6 +46,20 @@ export function RunPreviewPanel() { return 5_000; }, }); + const { data: attempts = [] } = useQuery({ + queryKey: ["run-attempts", id], + queryFn: () => api.listRunAttempts(id!), + enabled: Boolean(id && runId), + }); + const run = request + ? { + ...request, + run: + (runId + ? attempts.find((attempt) => attempt._id === runId) + : undefined) ?? request.run, + } + : undefined; const { data: profile } = useQuery({ queryKey: ["profile", run?.profileId], @@ -37,7 +67,9 @@ export function RunPreviewPanel() { enabled: !!run?.profileId, }); - const closePanel = () => navigate({ pathname: "/runs", search: window.location.search }); + const closePanel = + onClose ?? + (() => navigate({ pathname: "/runs", search: window.location.search })); if (isLoading) { return ( @@ -79,7 +111,7 @@ export function RunPreviewPanel() { onClose={closePanel} headerActions={
- + - - - {hidden.map((item) => renderMenuItem(item))} - - - )} -
- ); -} - -/** Small OS-platform icon. Falls back to a text badge for unknown platforms. */ -function OsPlatformIcon({ platform }: { platform: string }) { - const p = platform.toLowerCase(); - const Icon = p === "darwin" || p === "macos" - ? Apple - : p === "win32" || p === "windows" - ? AppWindow - : p === "linux" - ? FaLinux - : null; - const label = p === "darwin" || p === "macos" - ? "macOS" - : p === "win32" || p === "windows" - ? "Windows" - : p === "linux" - ? "Linux" - : platform; - if (!Icon) { - return ( - - {platform} - - ); - } - return ( - - - - - - - {label} - - ); -} - /** * Aggregate progress bar used in grouped rows for the Status and Outcome * columns. Shows `count/total label` with three explicit segments: @@ -318,6 +258,7 @@ const COLUMN_OPTIONS: CustomizeColumnsOption[] = [ // growth-ecosystems/scope-project#127). { id: "status", label: "Status" }, { id: "outcome", label: "Outcome" }, + { id: "taxonomy", label: "Taxonomy" }, { id: "experiment", label: "Experiment" }, { id: "task", label: "Task" }, { id: "criteria", label: "Criteria" }, @@ -455,6 +396,7 @@ function NumericComparatorRow({ export function RunsList() { const navigate = useNavigate(); + const location = useLocation(); const queryClient = useQueryClient(); const strictAgentCapabilities = useStrictAgentCapabilities(); const detailOutlet = useOutlet(); @@ -1423,7 +1365,7 @@ export function RunsList() { width: "120px", sticky: "left", stickyOffset: "40px", - cell: (r) => , + cell: (r) => , }, { id: "experiment", @@ -1446,58 +1388,14 @@ export function RunsList() { id: "task", header: "Task", hidden: columnVisibility.isHidden("task"), - cell: (r) => - r.scenario?.task ? ( - - - {truncate(r.scenario.task, 60)} - - - ) : ( - — - ), + cell: (r) => , }, { id: "criteria", header: "Criteria", width: "180px", hidden: columnVisibility.isHidden("criteria"), - cell: (r) => { - const criteria = r.scenario?.criteria ?? []; - return criteria.length > 0 ? ( - ( - - - e.stopPropagation()} - > - {criterionId} - - - {criterionId} - - )} - renderMenuItem={(criterionId) => ( - - e.stopPropagation()} - > - {criterionId} - - - )} - /> - ) : ( - — - ); - }, + cell: (r) => , }, { id: "worker", @@ -1505,221 +1403,54 @@ export function RunsList() { sortable: true, width: "180px", hidden: columnVisibility.isHidden("worker"), - cell: (r) => { - return ( - - ); - }, + cell: (r) => , }, { id: "version", header: "Version", width: "110px", hidden: columnVisibility.isHidden("version"), - cell: (r) => - r.agentVersion ? ( - - - {r.agentVersion} - - {r.agentVersion} - - ) : ( - — - ), + cell: (r) => , }, { id: "os", header: "OS", width: "120px", hidden: columnVisibility.isHidden("os"), - cell: (r) => { - const os = r.run?.os; - if (!os) return —; - return ( - - - - - - - - {os.platform} {os.release} ({os.arch}) - - - ); - }, + cell: (r) => , }, { id: "mcp", header: "MCP", width: "180px", hidden: columnVisibility.isHidden("mcp"), - cell: (r) => { - const servers = r.mcpServers ?? []; - return servers.length > 0 ? ( - ( - - - e.stopPropagation()} - > - {slug} - - - {slug} - - )} - renderMenuItem={(slug) => ( - - e.stopPropagation()} - > - {slug} - - - )} - /> - ) : ( - — - ); - }, + cell: (r) => , }, { id: "skills", header: "Skills", width: "180px", hidden: columnVisibility.isHidden("skills"), - cell: (r) => { - const refs = (r.skillRevisions ?? r.skills ?? []) as string[]; - return refs.length > 0 ? ( - { - const skillName = ref.split("@")[0].split("/").pop() ?? ref; - const skillSlug = ref.split("@")[0]; - return ( - - - e.stopPropagation()} - > - {truncate(skillName, 20)} - - - {ref} - - ); - }} - renderMenuItem={(ref) => { - const skillName = ref.split("@")[0].split("/").pop() ?? ref; - const skillSlug = ref.split("@")[0]; - return ( - - e.stopPropagation()} - > - {skillName} - - - ); - }} - /> - ) : ( - — - ); - }, + cell: (r) => , }, { id: "extensions", header: "Extensions", width: "190px", hidden: columnVisibility.isHidden("extensions"), - cell: (r) => { - const ids = r.extensions ?? []; - return ids.length > 0 ? ( - { - const [qualifiedName, version] = id.split("@"); - const shortName = qualifiedName.split(".").pop() ?? id; - const extensionLabel = `${shortName}${version ? `@${version}` : ""}`; - return ( - - - e.stopPropagation()} - > - {truncate(extensionLabel, 20)} - - - {id} - - ); - }} - renderMenuItem={(id) => { - const [qualifiedName, version] = id.split("@"); - const shortName = qualifiedName.split(".").pop() ?? id; - const extensionLabel = `${shortName}${version ? `@${version}` : ""}`; - return ( - - e.stopPropagation()} - > - {truncate(extensionLabel, 20)} - - - ); - }} - /> - ) : ( - — - ); - }, + cell: (r) => , }, { id: "profile", header: "Profile", width: "220px", hidden: columnVisibility.isHidden("profile"), - cell: (r) => { - if (!r.profileId) return —; - - const profileLabel = profileNameById.get(r.profileId) ?? formatId(r.profileId); - - return ( -
- - - {profileLabel} - - {profileLabel} - -
- ); - }, + cell: (r) => ( + + ), }, { id: "priority", @@ -1727,41 +1458,21 @@ export function RunsList() { sortable: true, width: "80px", hidden: columnVisibility.isHidden("priority"), - cell: (r) => - r.priority != null ? ( - {r.priority} - ) : ( - — - ), + cell: (r) => , }, { id: "model", header: "Model", width: "180px", hidden: columnVisibility.isHidden("model"), - cell: (r) => - r.model ? ( - - - {r.model} - - {r.model} - - ) : ( - — - ), + cell: (r) => , }, { id: "effort", header: "Effort", width: "100px", hidden: columnVisibility.isHidden("effort"), - cell: (r) => - r.reasoningEffort ? ( - {r.reasoningEffort} - ) : ( - — - ), + cell: (r) => , }, { id: "status", @@ -1769,24 +1480,21 @@ export function RunsList() { sortable: true, width: "120px", hidden: columnVisibility.isHidden("status"), - cell: (r) => - r.run?.status ? ( - - ) : ( - — - ), + cell: (r) => , }, { id: "outcome", header: "Outcome", width: "120px", hidden: columnVisibility.isHidden("outcome"), - cell: (r) => (r.run?.outcome ? : —), + cell: (r) => , + }, + { + id: "taxonomy", + header: "Taxonomy", + width: "150px", + hidden: columnVisibility.isHidden("taxonomy"), + cell: (r) => , }, { id: "report", @@ -1808,14 +1516,7 @@ export function RunsList() { header: "Attempt", width: "80px", hidden: columnVisibility.isHidden("attempt"), - cell: (r) => { - const n = r.run?.attemptNumber; - return n != null ? ( - #{n} - ) : ( - — - ); - }, + cell: (r) => , }, { id: "turns", @@ -1916,7 +1617,12 @@ export function RunsList() { title="View run" onClick={(e) => { e.stopPropagation(); - navigate(`/runs/${r._id}`); + navigate(`/runs/${r._id}`, { + state: runNavigationState( + `${location.pathname}${location.search}`, + "Back to runs", + ), + }); }} > @@ -2289,12 +1995,22 @@ export function RunsList() { {groupBy === "none" ? ( - r._id} activeId={activeId} - onRowClick={(r) => navigate({ pathname: `/runs/${r._id}/preview`, search: window.location.search })} + onRowClick={(r) => + navigate( + { pathname: `/runs/${r._id}/preview`, search: window.location.search }, + { + state: runNavigationState( + `${location.pathname}${location.search}`, + "Back to runs", + ), + }, + ) + } selection={{ selectedIds, onToggle: (id) => toggleRowSelection(id), @@ -2310,15 +2026,35 @@ export function RunsList() { ? "No runs match the current filters." : "No runs yet. Submit one with the New Run button." } + pagination={{ + page: state.page, + pageSize: state.pageSize, + total: estimatedTotal, + hasNext: !!cursors.next, + hasPrev: state.page > 1, + itemLabel: "runs", + onPageChange: handlePageChange, + onPageSizeChange: state.setPageSize, + }} /> ) : (
- r._id} activeId={activeId} - onRowClick={(r) => navigate({ pathname: `/runs/${r._id}/preview`, search: window.location.search })} + onRowClick={(r) => + navigate( + { pathname: `/runs/${r._id}/preview`, search: window.location.search }, + { + state: runNavigationState( + `${location.pathname}${location.search}`, + "Back to runs", + ), + }, + ) + } selection={{ selectedIds, onToggle: (id) => toggleRowSelection(id), @@ -2799,6 +2535,16 @@ export function RunsList() { loading={isGroupsLoading} loadingRows={state.pageSize} emptyState="" + pagination={{ + page: state.page, + pageSize: state.pageSize, + total: estimatedTotal, + hasNext: !!cursors.next, + hasPrev: state.page > 1, + itemLabel: "groups", + onPageChange: handlePageChange, + onPageSizeChange: state.setPageSize, + }} /> {!isGroupsLoading && serverGroups.length === 0 && (
@@ -2809,16 +2555,6 @@ export function RunsList() { )}
)} - 1} - itemLabel={groupBy === "none" ? "runs" : "groups"} - />
diff --git a/apps/portal/src/pages/SubmitRun.test.tsx b/apps/portal/src/pages/SubmitRun.test.tsx index 14517fb21..1edf7ca87 100644 --- a/apps/portal/src/pages/SubmitRun.test.tsx +++ b/apps/portal/src/pages/SubmitRun.test.tsx @@ -24,6 +24,7 @@ vi.mock("@/lib/api", () => ({ listProfileVersions: vi.fn(async () => []), listRuns: vi.fn(async () => ({ data: [] })), listModels: vi.fn(async () => []), + listTaxonomies: vi.fn(async () => []), createExperiment: vi.fn(async () => ({ experimentId: "experiment-1" })), }, })); @@ -47,7 +48,7 @@ vi.mock("@/components/ReasoningEffortSelect", () => ({ ReasoningEffortSelect: () => null, useModelCapabilities: () => ({ capabilitiesMap: new Map(), - activeModelIds: [], + activeModelIds: ["gpt-6.1-sol"], capabilitiesLoaded: true, }), useReasoningEffort: () => ({ @@ -210,6 +211,71 @@ describe("SubmitRun legacy composition guidance", () => { }); describe("SubmitRun Experiment composition", () => { + it("submits a worker and model for a composed experiment", async () => { + vi.mocked(api.listAgents).mockResolvedValue([{ + _id: "coder-acp-copilot", + name: "GitHub Copilot CLI", + description: "Copilot worker", + modelProvider: "github-copilot", + supportedModels: ["gpt-6.1-sol"], + defaultModel: "gpt-6.1-sol", + available: true, + capabilities: {}, + versions: [{ + agentVersion: "copilot-dev", + workerVersion: "development", + components: {}, + gitCommit: "development", + buildTime: new Date().toISOString(), + imageTag: "development", + status: "active", + queueName: "queue-coder-acp-copilot", + createdAt: new Date().toISOString(), + }], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }]); + renderSubmitRun("experiment"); + + fireEvent.change(screen.getByLabelText("Name *"), { + target: { value: "Copilot comparison" }, + }); + fireEvent.change(screen.getByLabelText(/Task/), { + target: { value: "Build a hello-world CLI" }, + }); + fireEvent.change(screen.getByLabelText("Max iterations"), { + target: { value: "1" }, + }); + fireEvent.change(screen.getByLabelText("Occurrences"), { + target: { value: "1" }, + }); + + const submit = await screen.findByRole("button", { + name: /Create experiment with 1 run/, + }); + await waitFor(() => + expect((submit as HTMLButtonElement).disabled).toBe(false) + ); + fireEvent.click(submit); + + await waitFor(() => { + expect(api.createExperiment).toHaveBeenCalledWith({ + metadata: { name: "Copilot comparison" }, + reportTemplateIds: [], + request: expect.objectContaining({ + scenario: { + task: "Build a hello-world CLI", + criteria: [], + }, + worker: "coder-acp-copilot", + model: "gpt-6.1-sol", + agentVersion: "copilot-dev", + maxIterations: 1, + }), + }); + }); + }); + it("keeps the optional description collapsed and retains its value across toggles", async () => { renderSubmitRun("experiment"); diff --git a/apps/portal/src/pages/SubmitRun.tsx b/apps/portal/src/pages/SubmitRun.tsx index 39641502a..d4a6862e5 100644 --- a/apps/portal/src/pages/SubmitRun.tsx +++ b/apps/portal/src/pages/SubmitRun.tsx @@ -43,6 +43,7 @@ import { } from "@/components/ReasoningEffortSelect"; import { TaskPromptPicker } from "@/components/TaskPromptPicker"; import { useCommandEnter } from "@/hooks/useCommandEnter"; +import { useFeatureFlags } from "@/contexts/FeatureFlagContext"; import { useVisibleGates } from "@/hooks/useVisibleGates"; import { useStrictAgentCapabilities } from "@/hooks/useStrictAgentCapabilities"; import { KbdBadge } from "@/components/KbdBadge"; @@ -257,6 +258,7 @@ export function SubmitRun({ const [experimentDescription, setExperimentDescription] = useState(""); const [experimentReportTemplateIds, setExperimentReportTemplateIds] = useState([]); + const [taxonomyRevisionId, setTaxonomyRevisionId] = useState(""); const [experimentDescriptionOpen, setExperimentDescriptionOpen] = useState(false); @@ -303,6 +305,8 @@ export function SubmitRun({ const [generateDescription, setGenerateDescription] = useState(""); // ─── Queries ──────────────────────────────────────────────────────────── + const { isFeatureEnabledFailClosed } = useFeatureFlags(); + const taxonomeEnabled = isFeatureEnabledFailClosed("taxonome"); const { data: agents = [], isSuccess: agentsLoaded } = useQuery({ queryKey: ["agents", "include-deleted"], queryFn: () => api.listAgents({ includeDeleted: true }), @@ -325,6 +329,11 @@ export function SubmitRun({ const manualExperimentReportTemplates = experimentReportTemplates.filter( (template) => !template.trigger, ); + const { data: taxonomies = [] } = useQuery({ + queryKey: ["taxonomies"], + queryFn: api.listTaxonomies, + enabled: mode === "experiment" && taxonomeEnabled, + }); const { data: profileVersions = [] } = useQuery({ queryKey: ["profile-versions", selectedProfileId], @@ -752,6 +761,7 @@ export function SubmitRun({ : {}), }, reportTemplateIds: experimentReportTemplateIds, + ...(taxonomyRevisionId ? { taxonomyRevisionId } : {}), ...(request ? { request } : { empty: true }), }) : api.submitRun(request!), @@ -785,6 +795,43 @@ export function SubmitRun({ const baseProfileSpec = selectedProfileId ? (selectedProfileVersion ? `${selectedProfileId}@${selectedProfileVersion}` : selectedProfileId) : null; + const experimentProfileTargets = mode === "experiment" && inVariationMode + ? [ + ...(selectedBaseProfileVersion + ? [{ + profileId: baseProfileSpec ?? selectedProfileId!, + worker: selectedBaseProfileVersion.workerType, + model: selectedBaseProfileVersion.model, + ...(selectedBaseProfileVersion.reasoningEffort + ? { reasoningEffort: selectedBaseProfileVersion.reasoningEffort } + : {}), + ...(selectedBaseProfileVersion.agentVersion + ? { agentVersion: selectedBaseProfileVersion.agentVersion } + : {}), + }] + : []), + ...normalizedVariationDrafts.flatMap((draft) => { + const profile = profileList.find(candidate => candidate._id === draft.profileId); + const version = variationProfileVersions[draft.profileId] + ?.find(candidate => candidate.version === draft.profileVersion) + ?? profile?.version; + if (!version) return []; + return [{ + profileId: draft.profileVersion + ? `${draft.profileId}@${draft.profileVersion}` + : draft.profileId, + worker: version.workerType, + model: version.model, + ...(version.reasoningEffort + ? { reasoningEffort: version.reasoningEffort } + : {}), + ...(version.agentVersion + ? { agentVersion: version.agentVersion } + : {}), + }]; + }), + ] + : undefined; const request: RunCreateInput = { scenario: { task: task.trim(), criteria: pickedCriteria }, @@ -807,6 +854,9 @@ export function SubmitRun({ ? { ...(baseProfileSpec ? { profileId: baseProfileSpec } : {}), profileVariations, + ...(experimentProfileTargets + ? { experimentProfileTargets } + : {}), } : { ...(baseProfileSpec ? { profileId: baseProfileSpec } : {}), @@ -975,6 +1025,39 @@ export function SubmitRun({ +
+ {taxonomeEnabled && ( + <> + + +

+ The exact immutable revision is pinned to this Experiment. +

+ + )} +
HttpResponse.json(taxonomyDetailFixture), + ), + ], + }, + }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const ExactOrderedElements: Story = { + play: async ({ canvas }) => { + await expect( + (await canvas.findAllByText("team-taxonomy@r2"))[0], + ).toBeVisible(); + await expect(canvas.getAllByText("Prompt")[0]).toBeVisible(); + await expect(canvas.getAllByText("Deterministic")[0]).toBeVisible(); + await expect(canvas.getAllByText("Built-in · read-only")[0]).toBeVisible(); + await userEvent.click( + canvas.getAllByRole("button", { name: "Edit prompt" })[0], + ); + await expect(screen.getByLabelText("Prompt instruction")).toHaveValue( + "Original prompt", + ); + }, +}; + +export const DeletedHistorical: Story = { + args: { + taxonomyId: "taxonomy-root", + revisionId: "taxonomy-revision-1", + }, + parameters: { + msw: { + handlers: [ + http.get( + "/api/v1/taxonomies/taxonomy-root/detail", + () => HttpResponse.json({ + ...taxonomyDetailFixture, + taxonomy: { + ...taxonomyDetailFixture.taxonomy, + deletedAt: "2026-10-04T00:00:00.000Z", + }, + selectedRevision: { + ...taxonomyDetailFixture.selectedRevision, + _id: "taxonomy-revision-1", + revision: 1, + ref: "team-taxonomy@r1", + elements: [{ ref: "element-b@r1" }], + }, + elements: [taxonomyDetailFixture.elements[0]], + }), + ), + ], + }, + }, + play: async ({ canvas }) => { + await expect(await canvas.findByText("Historical")).toBeVisible(); + await expect(canvas.getByText("Deleted")).toBeVisible(); + await expect( + canvas.queryByRole("button", { name: "Edit prompt" }), + ).not.toBeInTheDocument(); + }, +}; diff --git a/apps/portal/src/pages/TaxonomyDetail.test.tsx b/apps/portal/src/pages/TaxonomyDetail.test.tsx new file mode 100644 index 000000000..f4e3c0868 --- /dev/null +++ b/apps/portal/src/pages/TaxonomyDetail.test.tsx @@ -0,0 +1,249 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + cleanup, + fireEvent, + render, + screen, + within, + waitFor, +} from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import type { TaxonomyDetailDocument } from "@/types"; + +const detail: TaxonomyDetailDocument = { + taxonomy: { + _id: "taxonomy-root", + projectId: "project-1", + id: "team-taxonomy", + name: "Team taxonomy", + description: "Team-owned taxonomy", + sourceType: "user", + revisionCounter: 2, + latestRevisionId: "taxonomy-revision-2", + latestRevision: 2, + createdAt: "2026-10-01T00:00:00.000Z", + }, + selectedRevision: { + _id: "taxonomy-revision-2", + taxonomyId: "taxonomy-root", + projectId: "project-1", + id: "team-taxonomy", + version: "1.2.0", + sourceType: "user", + revision: 2, + ref: "team-taxonomy@r2", + contentSha256: "2".repeat(64), + schema_version: "scope/taxonomy/v1", + surfaces: [], + elements: [{ ref: "element-b@r1" }, { ref: "element-a@r1" }], + createdAt: "2026-10-02T00:00:00.000Z", + }, + revisions: [ + { + _id: "taxonomy-revision-2", + revision: 2, + ref: "team-taxonomy@r2", + version: "1.2.0", + elementCount: 2, + contentSha256: "2".repeat(64), + createdAt: "2026-10-02T00:00:00.000Z", + isLatest: true, + }, + { + _id: "taxonomy-revision-1", + revision: 1, + ref: "team-taxonomy@r1", + version: "1.0.0", + elementCount: 1, + contentSha256: "1".repeat(64), + createdAt: "2026-10-01T00:00:00.000Z", + isLatest: false, + }, + ], + elements: [ + { + position: 0, + root: { + _id: "element-root-b", + projectId: "project-1", + id: "element-b", + name: "Element B", + sourceType: "user", + revisionCounter: 1, + latestRevision: 1, + createdAt: "2026-10-01T00:00:00.000Z", + }, + revision: { + _id: "element-revision-b", + elementId: "element-root-b", + projectId: "project-1", + id: "element-b", + label: "Element B", + description: "First authored element", + sourceType: "user", + revision: 1, + ref: "element-b@r1", + contentSha256: "b".repeat(64), + implementation: { type: "prompt", prompt: "Original prompt" }, + preconditions: {}, + output: { evidence_roles: {} }, + createdAt: "2026-10-01T00:00:00.000Z", + }, + }, + { + position: 1, + root: { + _id: "element-root-a", + projectId: "project-1", + id: "element-a", + name: "Element A", + sourceType: "builtin", + revisionCounter: 1, + latestRevision: 1, + createdAt: "2026-10-01T00:00:00.000Z", + }, + revision: { + _id: "element-revision-a", + elementId: "element-root-a", + projectId: "project-1", + id: "element-a", + label: "Element A", + sourceType: "builtin", + revision: 1, + ref: "element-a@r1", + contentSha256: "a".repeat(64), + implementation: { + type: "deterministic", + handler: "scope/example-v1", + }, + preconditions: {}, + output: { evidence_roles: {} }, + createdAt: "2026-10-01T00:00:00.000Z", + }, + }, + ], +}; + +const apiMocks = vi.hoisted(() => ({ + getTaxonomyDetail: vi.fn(), + createTaxonomyElementRevision: vi.fn(), + createTaxonomyRevision: vi.fn(), + copyTaxonomyRevision: vi.fn(), + deleteTaxonomy: vi.fn(), +})); + +vi.mock("@/lib/api", () => ({ api: apiMocks })); + +import { TaxonomyDetail } from "./TaxonomyDetail"; + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +function renderDetail( + path = "/taxonomies/taxonomy-root", + value: TaxonomyDetailDocument = detail, +) { + apiMocks.getTaxonomyDetail.mockResolvedValue(value); + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return render( + + + + } /> + + + , + ); +} + +describe("TaxonomyDetail", () => { + it("renders only exact referenced elements in authored order", async () => { + renderDetail(); + + expect((await screen.findAllByText("team-taxonomy@r2")).length) + .toBeGreaterThan(0); + const rows = screen.getAllByRole("row"); + expect(within(rows[1]!).getByText("Element B")).toBeTruthy(); + expect(within(rows[2]!).getByText("Element A")).toBeTruthy(); + expect(screen.queryByText("Unrelated element")).toBeNull(); + expect(screen.getAllByText("Deterministic").length).toBeGreaterThan(0); + expect(screen.getAllByText("Built-in · read-only").length) + .toBeGreaterThan(0); + }); + + it("creates an element revision and a taxonomy revision when editing a prompt", async () => { + apiMocks.createTaxonomyElementRevision.mockResolvedValue({ + ...detail.elements[0]!.revision, + _id: "element-revision-b-2", + revision: 2, + ref: "element-b@r2", + implementation: { type: "prompt", prompt: "Updated prompt" }, + }); + apiMocks.createTaxonomyRevision.mockResolvedValue({ + ...detail.selectedRevision, + _id: "taxonomy-revision-3", + revision: 3, + ref: "team-taxonomy@r3", + elements: [{ ref: "element-b@r2" }, { ref: "element-a@r1" }], + }); + renderDetail(); + + await screen.findByText("Team taxonomy"); + fireEvent.click( + screen.getAllByRole("button", { name: "Edit prompt" })[0]!, + ); + fireEvent.change(screen.getByLabelText("Prompt instruction"), { + target: { value: "Updated prompt" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Create new revisions" })); + + await waitFor(() => { + expect(apiMocks.createTaxonomyElementRevision).toHaveBeenCalledWith( + "element-root-b", + expect.objectContaining({ + implementation: { type: "prompt", prompt: "Updated prompt" }, + }), + ); + }); + expect(apiMocks.createTaxonomyRevision).toHaveBeenCalledWith( + "taxonomy-root", + expect.objectContaining({ + elements: [{ ref: "element-b@r2" }, { ref: "element-a@r1" }], + }), + ); + }); + + it("requests and marks a selected historical revision read-only", async () => { + const historical = { + ...detail, + selectedRevision: { + ...detail.selectedRevision, + _id: "taxonomy-revision-1", + revision: 1, + ref: "team-taxonomy@r1", + elements: [{ ref: "element-b@r1" }], + }, + elements: [detail.elements[0]!], + }; + renderDetail( + "/taxonomies/taxonomy-root?revision=taxonomy-revision-1", + historical, + ); + + expect(await screen.findByText("Historical")).toBeTruthy(); + expect(apiMocks.getTaxonomyDetail).toHaveBeenCalledWith( + "taxonomy-root", + "taxonomy-revision-1", + ); + expect(screen.queryByRole("button", { name: "Edit prompt" })).toBeNull(); + }); +}); diff --git a/apps/portal/src/pages/TaxonomyDetail.tsx b/apps/portal/src/pages/TaxonomyDetail.tsx new file mode 100644 index 000000000..d2aa20a1a --- /dev/null +++ b/apps/portal/src/pages/TaxonomyDetail.tsx @@ -0,0 +1,546 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { useMemo, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useNavigate, useParams, useSearchParams } from "react-router-dom"; +import { + ArrowLeft, + Copy, + History, + Network, + Pencil, + Trash2, +} from "lucide-react"; +import { api } from "@/lib/api"; +import type { + ResolvedTaxonomyElement, +} from "@/types"; +import { cn, formatDate } from "@/lib/utils"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Textarea } from "@/components/ui/textarea"; +import { DataTable, type DataTableColumn } from "@/components/list-layout"; +import { toast } from "sonner"; + +/** Exact taxonomy revision detail with ordered resolved element revisions. */ +export function TaxonomyDetail({ + taxonomyId: taxonomyIdOverride, + revisionId: revisionIdOverride, +}: { + taxonomyId?: string; + revisionId?: string; +} = {}) { + const { taxonomyId: routeTaxonomyId } = useParams(); + const [searchParams, setSearchParams] = useSearchParams(); + const taxonomyId = taxonomyIdOverride ?? routeTaxonomyId; + const revisionId = + revisionIdOverride ?? searchParams.get("revision") ?? undefined; + const navigate = useNavigate(); + const queryClient = useQueryClient(); + const [deleteOpen, setDeleteOpen] = useState(false); + const [editing, setEditing] = useState(); + const [prompt, setPrompt] = useState(""); + + const detailQuery = useQuery({ + queryKey: ["taxonomy-detail", taxonomyId, revisionId ?? "latest"], + queryFn: () => api.getTaxonomyDetail(taxonomyId!, revisionId), + enabled: Boolean(taxonomyId), + }); + const detail = detailQuery.data; + const isHistorical = Boolean( + detail && + detail.selectedRevision._id !== detail.taxonomy.latestRevisionId, + ); + + const remove = useMutation({ + mutationFn: () => api.deleteTaxonomy(taxonomyId!), + onSuccess: () => { + void queryClient.invalidateQueries({ queryKey: ["taxonomies"] }); + toast.success("Taxonomy deleted"); + navigate("/taxonomies"); + }, + onError: mutationError => + toast.error( + mutationError instanceof Error + ? mutationError.message + : "Deletion failed", + ), + }); + const copy = useMutation({ + mutationFn: async () => { + if (!detail) throw new Error("Taxonomy detail is not loaded"); + return api.copyTaxonomyRevision(detail.selectedRevision._id, { + name: `${detail.taxonomy.name} copy`, + id: `${detail.taxonomy.id}-copy`, + }); + }, + onSuccess: created => { + void queryClient.invalidateQueries({ queryKey: ["taxonomies"] }); + toast.success("Built-in taxonomy copied"); + navigate(`/taxonomies/${created._id}`); + }, + onError: mutationError => + toast.error( + mutationError instanceof Error ? mutationError.message : "Copy failed", + ), + }); + const revisePrompt = useMutation({ + mutationFn: async () => { + if (!detail || !editing) throw new Error("No prompt element selected"); + if (editing.revision.implementation.type !== "prompt") { + throw new Error("Deterministic elements are read-only"); + } + const elementRevision = await api.createTaxonomyElementRevision( + editing.root._id, + { + schema_version: editing.revision.schema_version, + id: editing.revision.id, + label: editing.revision.label, + ...(editing.revision.description + ? { description: editing.revision.description } + : {}), + ...(editing.revision.category_id + ? { category_id: editing.revision.category_id } + : {}), + implementation: { type: "prompt", prompt }, + preconditions: editing.revision.preconditions, + output: editing.revision.output, + }, + ); + return api.createTaxonomyRevision(detail.taxonomy._id, { + schema_version: detail.selectedRevision.schema_version, + id: detail.selectedRevision.id, + version: detail.selectedRevision.version, + surfaces: detail.selectedRevision.surfaces, + elements: detail.selectedRevision.elements.map(reference => + "ref" in reference && reference.ref === editing.revision.ref + ? { ref: elementRevision.ref } + : reference + ), + }); + }, + onSuccess: revision => { + setEditing(undefined); + setPrompt(""); + void queryClient.invalidateQueries({ queryKey: ["taxonomies"] }); + void queryClient.invalidateQueries({ + queryKey: ["taxonomy-detail", taxonomyId], + }); + void queryClient.invalidateQueries({ queryKey: ["taxonomy-elements"] }); + setSearchParams({}); + toast.success(`Prompt updated in ${revision.ref}`); + }, + onError: mutationError => + toast.error( + mutationError instanceof Error + ? mutationError.message + : "Prompt update failed", + ), + }); + + const columns = useMemo[]>(() => [ + { + id: "position", + header: "#", + width: "48px", + cell: element => ( + + {element.position + 1} + + ), + hiddenOnCard: true, + }, + { + id: "label", + header: "Element", + cell: element => ( +
+

{element.revision.label}

+

+ {element.revision.ref} +

+
+ ), + }, + { + id: "implementation", + header: "Implementation", + width: "140px", + cell: element => ( + + {element.revision.implementation.type === "prompt" + ? "Prompt" + : "Deterministic"} + + ), + }, + { + id: "source", + header: "Source", + width: "150px", + cell: element => ( + + {element.root.sourceType === "builtin" + ? "Built-in · read-only" + : element.root.deletedAt ? "User · deleted" : "User"} + + ), + }, + { + id: "description", + header: "Description / category", + cell: element => ( +
+

{element.revision.description ?? "—"}

+ {element.revision.category_id && ( +

+ {element.revision.category_id} +

+ )} +
+ ), + }, + { + id: "revision", + header: "Exact revision", + width: "110px", + cell: element => ( + r{element.revision.revision} + ), + }, + { + id: "actions", + header: "", + width: "96px", + align: "right", + cell: element => { + const editable = + !isHistorical && + !detail?.taxonomy.deletedAt && + element.root.sourceType === "user" && + !element.root.deletedAt && + element.revision.implementation.type === "prompt"; + return editable ? ( + + ) : ( + Read-only + ); + }, + }, + ], [detail?.taxonomy.deletedAt, isHistorical]); + + if (detailQuery.isLoading) { + return ( +
+ + +
+ ); + } + + if (detailQuery.error || !detail) { + return ( +
+ +
+

Taxonomy not found

+

+ It may be deleted, outside the active project, or the selected + revision may not exist. +

+
+
+ ); + } + + return ( +
+ + +
+
+
+ +

+ {detail.taxonomy.name} +

+ + {detail.taxonomy.sourceType === "builtin" + ? "Built-in · read-only" + : "User"} + + {isHistorical && Historical} + {detail.taxonomy.deletedAt && ( + Deleted + )} +
+
+ {detail.selectedRevision.ref} + + semantic version {detail.selectedRevision.version} + + + {detail.elements.length} element + {detail.elements.length === 1 ? "" : "s"} + +
+ {detail.taxonomy.description && ( +

+ {detail.taxonomy.description} +

+ )} +
+
+ {detail.taxonomy.sourceType === "builtin" ? ( + + ) : ( + + )} +
+
+ +
+ + + Taxonomy elements + + Exact element revisions referenced by {detail.selectedRevision.ref}, + in authored order. Unrelated project elements are not included. + + + + `${element.position}:${element.revision._id}`} + emptyState="This taxonomy revision does not reference any elements." + density="compact" + /> + + + +
+ + + Revision identity + + + {detail.selectedRevision.ref} + + {detail.selectedRevision.version} + + + + {detail.selectedRevision.contentSha256} + + + + {formatDate(detail.selectedRevision.createdAt)} + + {detail.taxonomy.updatedAt && ( + + {formatDate(detail.taxonomy.updatedAt)} + + )} + + + + + + + Revision history + + + Selecting a historical revision resolves its exact element-ref + vector. + + + +
+ {detail.revisions.map(revision => ( + + ))} +
+
+
+
+
+ + { + if (!open) { + setEditing(undefined); + setPrompt(""); + } + }} + > + + + Edit {editing?.revision.label} prompt + + Creates a new element revision and taxonomy revision while + preserving prior exact refs. + + +
+

+ Creates a new element revision and a new taxonomy revision that + pins it. Prior revisions remain unchanged. +

+